diff options
| author | Anton <anton@picapica.im> | 2026-09-13 19:53:28 +0400 |
|---|---|---|
| committer | Anton <anton@picapica.im> | 2026-09-13 19:53:28 +0400 |
| commit | 9a84c95af6771590d4e5fe436e77bbe92a9dea92 (patch) | |
| tree | 71511a89b296d02d4106f9f031de806c8db422b7 /pica-client | |
| parent | e62dd7d30bec7a679013dcce8ec4bc2b2f842109 (diff) | |
| download | pica-pica-9a84c95af6771590d4e5fe436e77bbe92a9dea92.tar.gz pica-pica-9a84c95af6771590d4e5fe436e77bbe92a9dea92.zip | |
pica-client: use webrtc-audio-processing instead of speexdsp for echo cancellation
Diffstat (limited to 'pica-client')
| -rw-r--r-- | pica-client/audiodevice.cpp | 62 | ||||
| -rw-r--r-- | pica-client/audiodevice.h | 20 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 255 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.h | 79 | ||||
| -rw-r--r-- | pica-client/capturegain.cpp | 533 | ||||
| -rw-r--r-- | pica-client/capturegain.h | 81 | ||||
| -rw-r--r-- | pica-client/capturegain_coreaudio.cpp | 290 | ||||
| -rw-r--r-- | pica-client/capturegain_wasapi.cpp | 168 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 125 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.h | 16 | ||||
| -rw-r--r-- | pica-client/echocanceller.cpp | 492 | ||||
| -rw-r--r-- | pica-client/echocanceller.h | 174 | ||||
| -rw-r--r-- | pica-client/main.cpp | 13 | ||||
| -rw-r--r-- | pica-client/nativeaudio.h | 18 | ||||
| -rw-r--r-- | pica-client/nativeaudio_coreaudio.cpp | 109 | ||||
| -rw-r--r-- | pica-client/nativeaudio_wasapi.cpp | 10 | ||||
| -rw-r--r-- | pica-client/pica-client.pro | 38 |
17 files changed, 761 insertions, 1722 deletions
diff --git a/pica-client/audiodevice.cpp b/pica-client/audiodevice.cpp index e04aca9..fdbeedc 100644 --- a/pica-client/audiodevice.cpp +++ b/pica-client/audiodevice.cpp @@ -80,6 +80,13 @@ static const int64_t kOpusBitrate = 32000; static QMutex s_driverMutex; static QString s_linuxDriver = QStringLiteral("alsa"); +// The configured echo cancellation, published to the audio threads the same +// way and for the same reason. The initial value only stands until main() +// has read the setting, and is the one that cannot make things worse if it +// somehow does not. +static QMutex s_echoCancellationMutex; +static EchoCancellationMode s_echoCancellation = EchoCancellationNone; + static QString ff_errstr(int err) { char buf[256] = {0}; @@ -612,8 +619,16 @@ void AudioDevice::captureNative(const QString &api) return; } + // Ask the platform for its voice processing only when it is the one + // meant to be cancelling. Either of the other two settings wants the + // microphone as the hardware delivers it: EchoCanceller cannot model an + // echo path that something below it is already subtracting from, and + // "off" means off. + const NativeAudioMode mode = (AudioDevice::EchoCancellation() == EchoCancellationPlatform) + ? NativeAudioVoiceCall : NativeAudioVoiceCallRaw; + QString err; - if (!backend->openCapture(m_deviceName, m_sampleRate, kChannels, NativeAudioVoiceCall, &err)) + if (!backend->openCapture(m_deviceName, m_sampleRate, kChannels, mode, &err)) { qWarning() << err; emit errorOccurred(err); @@ -1021,8 +1036,15 @@ void AudioDevice::playNative(const QString &api) return; } + // Same choice as captureNative(), and it has to be the same answer: on + // macOS the platform canceller is one audio unit doing both directions, + // so asking for it here and not there would open a second unit on the + // same device. + const NativeAudioMode mode = (AudioDevice::EchoCancellation() == EchoCancellationPlatform) + ? NativeAudioVoiceCall : NativeAudioVoiceCallRaw; + QString err; - if (!backend->openPlayback(m_deviceName, m_sampleRate, kChannels, NativeAudioVoiceCall, &err)) + if (!backend->openPlayback(m_deviceName, m_sampleRate, kChannels, mode, &err)) { qWarning() << err; emit errorOccurred(err); @@ -1411,6 +1433,42 @@ QString AudioDevice::LinuxDriverName() return s_linuxDriver; } +void AudioDevice::SetEchoCancellation(EchoCancellationMode mode) +{ + QMutexLocker locker(&s_echoCancellationMutex); + s_echoCancellation = mode; +} + +EchoCancellationMode AudioDevice::EchoCancellation() +{ + QMutexLocker locker(&s_echoCancellationMutex); + return s_echoCancellation; +} + +bool AudioDevice::PlatformEchoCancellationAvailable(const QString &driver) +{ + // Windows and macOS both have one, and both are reached through the + // native backends rather than through libavdevice - which is why those + // backends exist at all. + if (AudioDevice::IsPlatformDriver(driver)) + return NativeAudioBackend::isAvailable(AudioDevice::PlatformApiName(driver)); + + // On Linux nothing is provided per stream. What there is instead is a + // sound server that can be made to hand out an already cancelled source - + // PulseAudio's module-echo-cancel, or PipeWire's equivalent - which the + // user then picks from the microphone list like any other device. Raw + // ALSA has nothing of the sort. + if (driver == QLatin1String("pulse")) + return true; + + return false; +} + +bool AudioDevice::PlatformEchoCancellationAvailable() +{ + return PlatformEchoCancellationAvailable(AudioDevice::PlatformDriverName(CAPTURE)); +} + QString AudioDevice::PlatformDriverName(enum MediaDeviceStreamDirection dir) { Q_UNUSED(dir) diff --git a/pica-client/audiodevice.h b/pica-client/audiodevice.h index 6b2db1c..b98acf2 100644 --- a/pica-client/audiodevice.h +++ b/pica-client/audiodevice.h @@ -55,7 +55,7 @@ public: // far end signal; both directions of one call must be given the same // object or there is nothing to cancel against. // - // Ignored when the platform is doing its own cancellation - see + // Null unless EchoCancellation() is EchoCancellationOwn - see // AudioVideoCallController::startAudioPipeline(), which is where that is // decided. void setEchoCanceller(EchoCancellerPtr ec); @@ -151,6 +151,24 @@ public: static void SetLinuxDriverName(const QString &driver); static QString LinuxDriverName(); + // Whether anything below us would cancel the acoustic echo, so that + // EchoCancellationPlatform is worth offering at all. + // + // The overload taking a driver answers for that driver rather than the + // configured one, which is what the settings dialog needs: on Linux the + // answer depends on the audio system, and the radio button has to follow + // the combo box before either has been applied. + static bool PlatformEchoCancellationAvailable(const QString &driver); + static bool PlatformEchoCancellationAvailable(); + + // Which of the three cancellers a call should use. Cached in a process + // wide variable for the same reason LinuxDriverName() is - the capture + // and playback loops run on their own threads and cannot reach the + // settings database. Set it from the GUI thread at startup and whenever + // the setting changes. + static void SetEchoCancellation(EchoCancellationMode mode); + static EchoCancellationMode EchoCancellation(); + private: QString m_deviceName; QString m_codec; diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index 405746d..72b6fc2 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -30,12 +30,10 @@ static const quint16 kVideoHeight = 480; // Rate we capture and encode at, announced to the peer in the 0x74 message. // -// 16kHz rather than 48kHz because of the echo canceller. speexdsp's MDF is -// tuned for 8/16kHz, and its cost is counted in taps: a 150ms tail is 7200 -// taps at 48kHz but only 2400 at 16kHz. Convergence time and steady state -// misadjustment both scale with that length, so the 48kHz filter spent every -// call chasing and never settling - visible in the AEC log as ERLE climbing -// towards 20dB and being knocked back to zero by each doubletalk burst. +// 16kHz rather than 48kHz because it is what the echo canceller wants. AEC3 +// does its filtering in the 0-8kHz band whatever it is handed; at 48kHz it +// splits the signal into four bands, cancels in the lowest and gates the rest, +// which is more work for a result that is no better on speech. // // Costs nothing worth having for speech: Opus codes this as wideband, an 8kHz // audio bandwidth, which is well past telephone quality and past where voice @@ -49,10 +47,7 @@ static const int kCallSampleRate = 16000; AudioVideoCallController::AudioVideoCallController(QObject *parent) : QObject(parent), callwindow(0), ringdevice(0), is_active(false), - m_captureGain(nullptr), m_savedCaptureGain(0.0), m_haveSavedCaptureGain(false), - m_gainCalibrating(false), m_captureGainSteps(0), m_gainStartDb(0.0), - m_haveGainDb(false), m_gainIneffectiveSteps(0), m_lastPinnedPercent(0.0), - m_gainSettleReports(0), m_audioSeq(0), + m_audioSeq(0), m_videoSeq(0), m_videoFrameTimestamp(0), m_videoFrameStarted(false) { connect(skynet, SIGNAL(IncomingCall(QByteArray)), this, SLOT(call_from(QByteArray))); @@ -189,11 +184,11 @@ void AudioVideoCallController::startAudioPipeline() // stopAudioPipeline() does not join either thread, so this has to outlive // whichever of them returns last. // - // It is created even on the platforms that cancel for us, because whether - // they really will is not known until the capture stream has been opened - // - see AudioDevice::captureNative(), which drops it once the backend - // says the platform has the job in hand. - if (st.loadValue("audio.echo_cancel", 1).toBool()) + // Only for EchoCancellationOwn. The other two settings mean something + // below us is doing the job or nobody is, and either way running this as + // well would only take a second bite out of the speech - the devices are + // opened accordingly, see AudioDevice::captureNative(). + if (AudioDevice::EchoCancellation() == EchoCancellationOwn) m_echoCanceller = EchoCancellerPtr(new EchoCanceller(kCallSampleRate)); else m_echoCanceller.clear(); @@ -206,10 +201,6 @@ void AudioVideoCallController::startAudioPipeline() microphone->setEchoCanceller(m_echoCanceller); output->setEchoCanceller(m_echoCanceller); - // Before Capture() starts, so the gain is already read and saved by the - // time the first clipping report can come back. - startCaptureGainControl(capDev); - // Declare our outgoing codec to the peer, then start capturing and // encoding right away - the peer's decoder is only ready once it has // processed this 0x74 message, but since the call is carried over TCP, @@ -238,12 +229,6 @@ void AudioVideoCallController::stopAudioPipeline() microphone->Close(); output->Close(); - // Puts the user's gain back. Safe to do while the capture thread is still - // winding down: the control is only ever touched from this thread, and - // capture_clipping() finds m_captureGain null and does nothing if a last - // queued report arrives after this. - stopCaptureGainControl(); - // Drop our reference. The two audio threads still hold theirs until they // come out of Capture()/Play(), and the canceller goes away with the last // of them. @@ -469,222 +454,20 @@ void AudioVideoCallController::send_audio_packet(QByteArray data) } // Rate limited by AudioDevice, which reports at most once every couple of -// seconds of audio, so this can log and act unconditionally. +// seconds of audio, so this can log unconditionally. +// +// Only a warning, and deliberately: clipping happens in the converter, before +// anything here sees a sample, so nothing in this process can undo it. The +// fix is less gain ahead of the ADC, which is the capture device's own mixer +// control and the user's to set. void AudioVideoCallController::capture_clipping(double pinnedPercent, double peakDb) { - qWarning("Microphone is clipping: %.2f%% of samples at full scale, peak %.1f dBFS", + qWarning("Microphone is clipping: %.2f%% of samples at full scale, peak %.1f dBFS - " + "turn the recording level down, it distorts the speech and stops the echo " + "canceller converging", pinnedPercent, peakDb); - - // Either the setting is off, or nothing on this platform would let us at - // the control. The warning above is then all we can do - and it is still - // worth having, since the distortion cannot be undone anywhere else. - if (!m_captureGain || !m_gainCalibrating) - return; - - // Calibration is a thing that happens at the start of a call, not a gain - // rider. Once the window has passed the level is whatever it is: moving it - // mid-conversation would keep dragging the echo path out from under the - // canceller, which is worse than a bit of distortion. - if (m_callClock.elapsed() > kGainCalibrationMs) - { - closeGainCalibration("calibration window has passed"); - return; - } - - // The audio in this report may predate the last adjustment. - if (m_gainSettleReports > 0) - { - m_gainSettleReports--; - return; - } - - // Without a dB scale there is no safe way to know how far down we have - // come, so the step count has to be the bound - and a much tighter one, - // since six steps of 0.7 is over 50 dB on a perceptual scale. - const int stepLimit = m_haveGainDb ? kMaxCaptureGainSteps : kMaxBlindGainSteps; - if (m_captureGainSteps >= stepLimit) - { - closeGainCalibration("step limit reached"); - return; - } - - // Did the previous adjustment achieve anything? Two that did not means the - // stage overloading is not the one we are holding. - if (m_captureGainSteps > 0 && pinnedPercent >= m_lastPinnedPercent) - { - m_gainIneffectiveSteps++; - if (m_gainIneffectiveSteps >= kMaxGainIneffectiveSteps) - { - closeGainCalibration("lowering this control is not reducing the clipping - " - "something ahead of it (an analog mic boost, or Windows' " - "separate Microphone Boost) is what is overloading"); - return; - } - } - else - { - m_gainIneffectiveSteps = 0; - } - - double current = 0.0; - if (!m_captureGain->gain(¤t)) - return; - - // The bound that matters, and the one an earlier version of this got - // wrong: how far down we have come in decibels, not where the slider sits. - double currentDb = 0.0; - if (m_haveGainDb && m_captureGain->gainDb(¤tDb)) - { - if (m_gainStartDb - currentDb >= kMaxGainReductionDb) - { - closeGainCalibration(QString("already %1 dB down and still clipping") - .arg(m_gainStartDb - currentDb, 0, 'f', 1)); - return; - } - } - - // Size the step to how badly it is clipping. A microphone left at maximum - // is 20-30dB too hot and wants covering quickly; one that only tips over - // on the loudest syllable wants nudging, not slamming. - double factor; - if (pinnedPercent >= 5.0) - factor = 0.70; - else if (pinnedPercent >= 0.5) - factor = 0.85; - else - factor = 0.95; - - double next = current * factor; - if (next >= current) - { - closeGainCalibration("control is already at its minimum"); - return; - } - - if (!m_captureGain->setGain(next)) - { - closeGainCalibration("the control would not move"); - return; - } - - m_captureGainSteps++; - m_lastPinnedPercent = pinnedPercent; - m_gainSettleReports = kGainSettleReports; - - double achieved = 0.0; - double newDb = 0.0; - if (m_haveGainDb && m_captureGain->gainDb(&newDb)) - { - achieved = currentDb - newDb; - qDebug("Capture gain %.0f%% -> %.0f%% (%.1f dB, %.1f dB below the user's setting)", - current * 100.0, next * 100.0, -achieved, m_gainStartDb - newDb); - - // A step that moved the slider but not the level means we are pushing - // against the end of a control that no longer does anything. - if (achieved < kMinEffectiveStepDb) - m_gainIneffectiveSteps++; - - // Close on the step that crosses the ceiling rather than waiting for - // another report, which would let one more step through first. - if (m_gainStartDb - newDb >= kMaxGainReductionDb) - closeGainCalibration(QString("reached the %1 dB limit") - .arg(kMaxGainReductionDb, 0, 'f', 0)); - } - else - { - qDebug("Capture gain %.0f%% -> %.0f%% (step %d of %d)", - current * 100.0, next * 100.0, m_captureGainSteps, kMaxCaptureGainSteps); - } - - // The echo path just changed by however many dB that was, so everything - // the filter has learned about it is now wrong. Telling it to start over - // costs a second of convergence; not telling it costs the rest of the call. - if (m_echoCanceller) - m_echoCanceller->reset(); -} - -void AudioVideoCallController::closeGainCalibration(const QString &why) -{ - if (!m_gainCalibrating) - return; - - m_gainCalibrating = false; - qDebug() << "Microphone gain calibration finished:" << why; -} - -void AudioVideoCallController::startCaptureGainControl(const QString &captureDevice) -{ - m_captureGain = nullptr; - m_savedCaptureGain = 0.0; - m_haveSavedCaptureGain = false; - m_captureGainSteps = 0; - m_gainCalibrating = false; - m_gainStartDb = 0.0; - m_haveGainDb = false; - m_gainIneffectiveSteps = 0; - m_lastPinnedPercent = 0.0; - m_gainSettleReports = 0; - - Settings st(config_dbname); - if (!st.loadValue("audio.auto_capture_gain", 0).toBool()) - return; - - // Whichever driver the capture side is about to open the device with - - // the mixer control has to be the one belonging to that device. - const QString driver = AudioDevice::PlatformDriverName(CAPTURE); - - m_captureGain = CaptureGainControl::create(driver, captureDevice); - if (!m_captureGain) - return; - - if (!m_captureGain->gain(&m_savedCaptureGain)) - { - // Without a reading there is nothing to put back afterwards, and - // moving a control we cannot restore is not a trade worth making. - qWarning() << "Could not read the capture gain - not taking it over"; - delete m_captureGain; - m_captureGain = nullptr; - return; - } - - m_haveSavedCaptureGain = true; - m_haveGainDb = m_captureGain->gainDb(&m_gainStartDb); - m_gainCalibrating = true; - - if (m_haveGainDb) - qDebug("Microphone gain calibration open on %s - at %.0f%% (%.1f dB), will cut at most %.0f dB", - qPrintable(m_captureGain->description()), m_savedCaptureGain * 100.0, - m_gainStartDb, kMaxGainReductionDb); - else - qDebug("Microphone gain calibration open on %s - at %.0f%%, no dB scale so bounded " - "to %d steps", - qPrintable(m_captureGain->description()), m_savedCaptureGain * 100.0, - kMaxCaptureGainSteps); } -void AudioVideoCallController::stopCaptureGainControl() -{ - if (!m_captureGain) - return; - - // Put the user's own setting back. Only bother if we actually moved it, - // so a call that never clipped cannot disturb a gain the user changed - // themselves while it was running. - if (m_haveSavedCaptureGain && m_captureGainSteps > 0) - { - if (m_captureGain->setGain(m_savedCaptureGain)) - qDebug() << "Capture gain restored to" << qRound(m_savedCaptureGain * 100.0) << "%"; - else - qWarning() << "Could not restore the capture gain to" - << qRound(m_savedCaptureGain * 100.0) << "% - it has been left turned down"; - } - - delete m_captureGain; - m_captureGain = nullptr; - m_haveSavedCaptureGain = false; - m_captureGainSteps = 0; - m_gainCalibrating = false; -} void AudioVideoCallController::incoming_audio_params(QByteArray peer_id, QString codec, quint16 sample_rate) { diff --git a/pica-client/audiovideocallcontroller.h b/pica-client/audiovideocallcontroller.h index f7c9e37..0743601 100644 --- a/pica-client/audiovideocallcontroller.h +++ b/pica-client/audiovideocallcontroller.h @@ -22,7 +22,6 @@ #include <QElapsedTimer> #include "callwindow.h" #include "audiodevice.h" -#include "capturegain.h" #include "videodevice.h" #include "toneplayer/toneplayer.h" @@ -75,75 +74,10 @@ private: QThread remotevideo_thread; // Acoustic echo cancellation for the current call, shared by microphone - // and output. Null when it is switched off in the settings, when - // speexdsp could not start one, or between calls. + // and output. Null unless the "audio.echo_cancellation" setting is + // EchoCancellationOwn, and between calls. EchoCancellerPtr m_echoCanceller; - // The capture device's mixer control, held for the duration of a call so - // a clipping microphone can be turned down. Null when the setting is off, - // when the platform has no reachable control, or between calls. - // - // m_savedCaptureGain is where the control was found at the start of the - // call, put back in stopAudioPipeline(): this is the user's own setting - // and we are only borrowing it. - CaptureGainControl *m_captureGain; - double m_savedCaptureGain; - bool m_haveSavedCaptureGain; - - // This is a calibration that runs at the start of a call and then stops - - // NOT a gain control that rides the level for the call's duration. The - // difference matters: an adaptive echo canceller can only track a - // stationary echo path, and moving the microphone gain moves that path. - // See the AGC comment in EchoCanceller's constructor, which says the same - // thing about the reason speexdsp's own AGC is left switched off. - // - // So every adjustment resets the canceller, and once m_gainCalibrating is - // false nothing touches the control again until the call ends. - bool m_gainCalibrating; - int m_captureGainSteps; - - // dB reading when calibration opened, and whether the platform would give - // us one. Bounding the total cut in decibels rather than as a fraction of - // the control's travel is the whole point: a fraction means different - // things on a hardware mixer (linear in dB) and on a sound server or - // WASAPI endpoint (roughly cubic), which is how an earlier version of this - // managed to wind a microphone down to silence. - double m_gainStartDb; - bool m_haveGainDb; - - // Consecutive adjustments that moved the control but did not reduce the - // clipping. Means the stage actually overloading is one we cannot reach - - // an analog mic boost ahead of the converter, or Windows' separate - // "Microphone Boost" - and that turning our control down further will - // silence the user without ever fixing it. - int m_gainIneffectiveSteps; - double m_lastPinnedPercent; - - // Reports to ignore after an adjustment, while audio captured at the old - // setting drains out of the pipeline. - int m_gainSettleReports; - - static const int kMaxCaptureGainSteps = 6; - - // Bound used when the control will not report decibels at all, where we - // are working blind and a wrong guess is what silences a microphone. - static const int kMaxBlindGainSteps = 2; - static const int kMaxGainIneffectiveSteps = 2; - static const int kGainSettleReports = 1; - - // How long the calibration window stays open, in milliseconds of call - // time. Long enough that someone who says nothing for the first few - // seconds still gets calibrated, short enough to be over before the - // conversation proper. - static const int kGainCalibrationMs = 20000; - - // Hard ceiling on the total cut, in dB below where the user had it. Past - // this the problem is not one we are going to fix by turning a knob. - static constexpr double kMaxGainReductionDb = 24.0; - - // An adjustment that achieves less than this much is not doing anything. - static constexpr double kMinEffectiveStepDb = 0.5; - // Sequence number / capture-clock timestamp stamped onto outgoing // 0x76 audio packets (see PICA_PROTO_CALL_PACKET_HDRSIZE). quint16 m_audioSeq; @@ -161,15 +95,6 @@ private: // Reassembly of incoming 0x77 fragments into whole encoded frames. VideoFrameAssembler m_videoAssembler; - // Take over the capture device's mixer gain for the duration of a call, - // and hand it back. Both are no-ops when "audio.auto_capture_gain" is off - // or the platform has no control we can reach. - void startCaptureGainControl(const QString &captureDevice); - void stopCaptureGainControl(); - - // Ends the calibration window, logging why. Idempotent. - void closeGainCalibration(const QString &why); - void playEarpieceTone(void (TonePlayer::*tone)()); void playRingTone(); void stopTones(); diff --git a/pica-client/capturegain.cpp b/pica-client/capturegain.cpp deleted file mode 100644 index 8f89718..0000000 --- a/pica-client/capturegain.cpp +++ /dev/null @@ -1,533 +0,0 @@ -/* - (c) Copyright 2012 - 2026 Anton Sviridenko - https://picapica.im - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, version 3. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. -*/ -#include "capturegain.h" - -#include <QDebug> -#include <QtGlobal> -#include <QStringList> -#include <cmath> - -#ifdef Q_OS_LINUX -#include <alsa/asoundlib.h> -#endif - -#ifdef HAVE_LIBPULSE -#include <pulse/pulseaudio.h> -#endif - -// Per platform factories, defined next to the implementations themselves - -// same arrangement as NativeAudioBackend::create(). -#ifdef Q_OS_WIN -CaptureGainControl *pica_create_wasapi_capture_gain(const QString &device); -#endif - -#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) -CaptureGainControl *pica_create_coreaudio_capture_gain(const QString &device); -#endif - -CaptureGainControl::~CaptureGainControl() -{ -} - -static double clampGain(double v) -{ - if (v < 0.0) - return 0.0; - if (v > 1.0) - return 1.0; - return v; -} - -// PulseAudio is the one backend whose control goes past its nominal maximum - -// PA_VOLUME_NORM is 100% and the server will happily sit above it. Clamping a -// reading to 1.0 there would mean handing a user who had their source at 130% -// back at 100% when the call ended, so that path keeps whatever it read and -// only guards the bottom of the range and a runaway top. -static double clampGainAmplified(double v) -{ - if (v < 0.0) - return 0.0; - if (v > 2.5) - return 2.5; - return v; -} - -#ifdef Q_OS_LINUX - -// The mixer lives on a card, but what we are handed is a PCM name. The two -// namespaces overlap only by convention, so this covers the forms that -// actually turn up in the device list and falls back to the default card for -// anything else - which is right for plain "default" and no worse than -// nothing for the exotic cases. -static QString alsaCardFromPcm(const QString &pcm) -{ - int card = pcm.indexOf(QLatin1String("CARD=")); - if (card >= 0) - { - QString name = pcm.mid(card + 5); - int comma = name.indexOf(QLatin1Char(',')); - if (comma >= 0) - name = name.left(comma); - if (!name.isEmpty()) - return QStringLiteral("hw:") + name; - } - - // "hw:1,0", "plughw:CARD=PCH,DEV=0", "dsnoop:1" - everything before the - // first comma after the colon names the card. - int colon = pcm.indexOf(QLatin1Char(':')); - if (colon >= 0) - { - QString rest = pcm.mid(colon + 1); - int comma = rest.indexOf(QLatin1Char(',')); - if (comma >= 0) - rest = rest.left(comma); - if (!rest.isEmpty()) - return QStringLiteral("hw:") + rest; - } - - return QStringLiteral("default"); -} - -class AlsaCaptureGain : public CaptureGainControl -{ -public: - AlsaCaptureGain() : m_mixer(nullptr), m_elem(nullptr), m_min(0), m_max(0) {} - - ~AlsaCaptureGain() override - { - if (m_mixer) - snd_mixer_close(m_mixer); - } - - bool open(const QString &pcmName) - { - // The card the PCM name points at, and then the real cards as - // fallbacks. "default" is not a card at all on a machine running a - // sound server: attaching to it lands on the server's own software - // volume, which is applied to samples the converter has already - // digitised. That control cannot undo clipping - scaling a clipped - // buffer down just gives a quieter clipped buffer - so calibrating - // against it would turn the microphone down forever without the - // clipping ever going away. Only a gain ahead of the ADC is any use - // here, and tryCard() insists on one. - QStringList candidates; - candidates << alsaCardFromPcm(pcmName); - - if (candidates.first() == QLatin1String("default")) - { - for (int card = 0; card < 8; card++) - candidates << (QStringLiteral("hw:") + QString::number(card)); - } - - for (int i = 0; i < candidates.size(); i++) - { - if (tryCard(candidates[i])) - return true; - } - - return false; - } - - bool gain(double *out) override - { - if (!m_elem || !out) - return false; - - // The kernel may have moved the control under us - another mixer app, - // or the user's own volume keys. - snd_mixer_handle_events(m_mixer); - - long v = 0; - if (snd_mixer_selem_get_capture_volume(m_elem, SND_MIXER_SCHN_FRONT_LEFT, &v) < 0 && - snd_mixer_selem_get_capture_volume(m_elem, SND_MIXER_SCHN_MONO, &v) < 0) - return false; - - *out = clampGain(double(v - m_min) / double(m_max - m_min)); - return true; - } - - bool gainDb(double *out) override - { - if (!m_elem || !out) - return false; - - snd_mixer_handle_events(m_mixer); - - // ALSA reports hundredths of a decibel. Not every element has a dB - // mapping - the software controls a sound server exposes often do not. - long v = 0; - if (snd_mixer_selem_get_capture_dB(m_elem, SND_MIXER_SCHN_FRONT_LEFT, &v) < 0 && - snd_mixer_selem_get_capture_dB(m_elem, SND_MIXER_SCHN_MONO, &v) < 0) - return false; - - *out = double(v) / 100.0; - return true; - } - - bool setGain(double value) override - { - if (!m_elem) - return false; - - long v = m_min + (long)llround(clampGain(value) * double(m_max - m_min)); - return snd_mixer_selem_set_capture_volume_all(m_elem, v) >= 0; - } - - QString description() const override { return m_name; } - -private: - // A candidate has to have a decibel range as well as a volume range. That - // is what separates a converter gain from a sound server's software fader: - // the hardware control knows what its steps are worth in dB, the software - // one only has a slider position. It is also what lets the caller bound - // the total cut in dB, which is the only unit that means the same thing on - // every backend. - bool tryCard(const QString &card) - { - if (m_mixer) - { - snd_mixer_close(m_mixer); - m_mixer = nullptr; - m_elem = nullptr; - } - - if (snd_mixer_open(&m_mixer, 0) < 0) - { - m_mixer = nullptr; - return false; - } - - const QByteArray cardUtf8 = card.toUtf8(); - if (snd_mixer_attach(m_mixer, cardUtf8.constData()) < 0 || - snd_mixer_selem_register(m_mixer, nullptr, nullptr) < 0 || - snd_mixer_load(m_mixer) < 0) - return false; - - // "Capture" is what the recording gain is called on essentially every - // card that has one; "Mic" turns up on the ones that do not separate - // the input selector from the gain. - m_elem = findNamed("Capture"); - if (!m_elem) - m_elem = findNamed("Mic"); - if (!m_elem) - m_elem = findNamed("Digital"); - - if (!m_elem) - return false; - - if (snd_mixer_selem_get_capture_volume_range(m_elem, &m_min, &m_max) < 0 || m_max <= m_min) - return false; - - long dbMin = 0, dbMax = 0; - if (snd_mixer_selem_get_capture_dB_range(m_elem, &dbMin, &dbMax) < 0 || dbMax <= dbMin) - { - m_elem = nullptr; - return false; - } - - m_name = QString::fromUtf8(snd_mixer_selem_get_name(m_elem)) + - QStringLiteral(" on ") + card; - return true; - } - - snd_mixer_elem_t *findNamed(const char *want) - { - for (snd_mixer_elem_t *e = snd_mixer_first_elem(m_mixer); e; e = snd_mixer_elem_next(e)) - { - if (!snd_mixer_selem_is_active(e) || !snd_mixer_selem_has_capture_volume(e)) - continue; - if (qstrcmp(snd_mixer_selem_get_name(e), want) == 0) - return e; - } - return nullptr; - } - - snd_mixer_t *m_mixer; - snd_mixer_elem_t *m_elem; - long m_min; - long m_max; - QString m_name; -}; - -#endif // Q_OS_LINUX - -#ifdef HAVE_LIBPULSE - -// PulseAudio's introspection and its setters are both asynchronous, so this -// keeps a connected context with a private mainloop and pumps it until each -// operation has called back. Same shape as enumeratePulse() in -// audiodevice.cpp, but long lived, since a call makes several of these. -class PulseCaptureGain : public CaptureGainControl -{ -public: - PulseCaptureGain() - : m_mainloop(nullptr), m_ctx(nullptr), m_channels(1), - m_pending(false), m_ok(false), m_value(0.0), m_valueDb(0.0) {} - - ~PulseCaptureGain() override - { - if (m_ctx) - { - pa_context_disconnect(m_ctx); - pa_context_unref(m_ctx); - } - if (m_mainloop) - pa_mainloop_free(m_mainloop); - } - - bool open(const QString &device) - { - m_mainloop = pa_mainloop_new(); - if (!m_mainloop) - return false; - - m_ctx = pa_context_new(pa_mainloop_get_api(m_mainloop), "pica-client"); - if (!m_ctx) - return false; - - if (pa_context_connect(m_ctx, nullptr, PA_CONTEXT_NOFLAGS, nullptr) < 0) - return false; - - // Five seconds at 50ms a turn, so a server that never answers cannot - // wedge the call setup. - for (int i = 0; i < 100; i++) - { - pa_context_state_t st = pa_context_get_state(m_ctx); - if (st == PA_CONTEXT_READY) - break; - if (st == PA_CONTEXT_FAILED || st == PA_CONTEXT_TERMINATED) - return false; - if (!pump()) - return false; - } - - if (pa_context_get_state(m_ctx) != PA_CONTEXT_READY) - return false; - - m_source = device; - if (m_source.isEmpty() || m_source == QLatin1String("default")) - { - if (!resolveDefaultSource()) - return false; - } - - // Also establishes the channel count, which setGain() needs. - double ignored; - return gain(&ignored); - } - - bool gain(double *out) override - { - if (!m_ctx || m_source.isEmpty()) - return false; - - const QByteArray name = m_source.toUtf8(); - m_pending = true; - m_ok = false; - - pa_operation *op = pa_context_get_source_info_by_name(m_ctx, name.constData(), - sourceInfoCb, this); - if (!await(op)) - return false; - - if (m_ok && out) - *out = m_value; - return m_ok; - } - - // pa_sw_volume_to_dB() returns -inf for a muted control, which is true but - // useless as a number to do arithmetic on. - bool gainDb(double *out) override - { - if (!out) - return false; - - double ignored; - if (!gain(&ignored)) - return false; - - if (m_valueDb < -200.0 || m_valueDb > 200.0) - return false; - - *out = m_valueDb; - return true; - } - - bool setGain(double value) override - { - if (!m_ctx || m_source.isEmpty()) - return false; - - pa_cvolume cv; - pa_cvolume_set(&cv, m_channels ? m_channels : 1, - (pa_volume_t)llround(clampGainAmplified(value) * double(PA_VOLUME_NORM))); - - const QByteArray name = m_source.toUtf8(); - m_pending = true; - m_ok = false; - - pa_operation *op = pa_context_set_source_volume_by_name(m_ctx, name.constData(), &cv, - successCb, this); - if (!await(op)) - return false; - - return m_ok; - } - - QString description() const override - { - return m_source + QStringLiteral(" (PulseAudio)"); - } - -private: - bool pump() - { - if (pa_mainloop_prepare(m_mainloop, 50 * 1000) < 0) - return false; - if (pa_mainloop_poll(m_mainloop) < 0) - return false; - if (pa_mainloop_dispatch(m_mainloop) < 0) - return false; - return true; - } - - // Runs the loop until the callback for op has fired, then reaps it. - bool await(pa_operation *op) - { - if (!op) - { - m_pending = false; - return false; - } - - for (int i = 0; i < 100 && m_pending; i++) - { - if (!pump()) - break; - } - - bool finished = !m_pending; - m_pending = false; - pa_operation_unref(op); - return finished; - } - - bool resolveDefaultSource() - { - m_pending = true; - m_ok = false; - - pa_operation *op = pa_context_get_server_info(m_ctx, serverInfoCb, this); - if (!await(op)) - return false; - - return m_ok && !m_source.isEmpty(); - } - - static void serverInfoCb(pa_context *, const pa_server_info *info, void *userdata) - { - PulseCaptureGain *self = (PulseCaptureGain *)userdata; - - if (info && info->default_source_name) - { - self->m_source = QString::fromUtf8(info->default_source_name); - self->m_ok = true; - } - self->m_pending = false; - } - - static void sourceInfoCb(pa_context *, const pa_source_info *info, int eol, void *userdata) - { - PulseCaptureGain *self = (PulseCaptureGain *)userdata; - - if (eol) - { - self->m_pending = false; - return; - } - - if (!info) - return; - - self->m_channels = info->volume.channels; - // The loudest channel, so a step down is a step down for all of them - // rather than only for whichever happens to be quietest. - self->m_value = clampGainAmplified(double(pa_cvolume_max(&info->volume)) / double(PA_VOLUME_NORM)); - self->m_valueDb = pa_sw_volume_to_dB(pa_cvolume_max(&info->volume)); - self->m_ok = true; - } - - static void successCb(pa_context *, int success, void *userdata) - { - PulseCaptureGain *self = (PulseCaptureGain *)userdata; - self->m_ok = (success != 0); - self->m_pending = false; - } - - pa_mainloop *m_mainloop; - pa_context *m_ctx; - QString m_source; - uint8_t m_channels; - - bool m_pending; - bool m_ok; - double m_value; - double m_valueDb; -}; - -#endif // HAVE_LIBPULSE - -CaptureGainControl *CaptureGainControl::create(const QString &driver, const QString &device) -{ -#ifdef Q_OS_WIN - if (driver == QLatin1String("platform-wasapi")) - return pica_create_wasapi_capture_gain(device); -#endif - -#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) - if (driver == QLatin1String("platform-coreaudio")) - return pica_create_coreaudio_capture_gain(device); -#endif - -#ifdef HAVE_LIBPULSE - if (driver == QLatin1String("pulse")) - { - PulseCaptureGain *g = new PulseCaptureGain(); - if (g->open(device)) - return g; - delete g; - qWarning() << "No PulseAudio volume control for capture source" << device; - return nullptr; - } -#endif - -#ifdef Q_OS_LINUX - if (driver == QLatin1String("alsa")) - { - AlsaCaptureGain *g = new AlsaCaptureGain(); - if (g->open(device)) - return g; - delete g; - qWarning() << "No ALSA capture volume control for device" << device; - return nullptr; - } -#endif - - Q_UNUSED(driver) - Q_UNUSED(device) - return nullptr; -} diff --git a/pica-client/capturegain.h b/pica-client/capturegain.h deleted file mode 100644 index a0d3bdc..0000000 --- a/pica-client/capturegain.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - (c) Copyright 2012 - 2026 Anton Sviridenko - https://picapica.im - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, version 3. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. -*/ -#ifndef CAPTUREGAIN_H -#define CAPTUREGAIN_H - -#include <QString> -#include <QtGlobal> - -// The capture device's own gain control - the mixer slider, not a multiplier -// applied to samples we already have. -// -// The distinction is the entire point of this class. Clipping happens in the -// converter, before any buffer reaches us; by then the peaks are flat and the -// harmonics they generated are in the signal for good. Scaling that buffer -// down produces a quieter clipped buffer and nothing else - in particular it -// does not help the echo canceller, whose adaptive filter is linear and -// therefore scale invariant. What defeats the filter is the non-linearity, and -// the only place to prevent one is ahead of the ADC. That is this control. -// -// Gain is expressed as a 0.0 to 1.0 fraction of the control's travel, which is -// what WASAPI and CoreAudio use natively and what ALSA and PulseAudio's ranges -// map onto cleanly. The mapping from that fraction to decibels is the -// platform's business and is not the same on all of them, so callers should -// treat a step as "somewhat quieter" and close the loop by measuring again -// rather than doing arithmetic in dB. -// -// Not thread safe, and deliberately so: AudioVideoCallController owns one of -// these and touches it only from the GUI thread, driven by a queued signal -// from the capture thread. Some of the implementations below need a message -// loop or a COM apartment and have no business being called from an audio -// callback. -class CaptureGainControl -{ -public: - virtual ~CaptureGainControl(); - - // Current position of the control, 0.0 to 1.0. False if it could not be - // read, in which case *out is untouched. - virtual bool gain(double *out) = 0; - - // The same position in decibels, where the platform can say. This is what - // a caller should bound itself by: the fraction above is a position on a - // slider whose relationship to loudness differs per backend - linear in dB - // on an ALSA hardware control, roughly cubic on PulseAudio and WASAPI - so - // "5% of travel" is a mild cut on one and near silence on another. Decibels - // mean the same thing everywhere. - // - // False when the platform will not say, and then the caller has nothing to - // go on but a step count. - virtual bool gainDb(double *out) { Q_UNUSED(out) return false; } - - // Moves the control. Values outside 0.0 to 1.0 are clamped. - virtual bool setGain(double value) = 0; - - // What was actually found - the mixer element or endpoint name - for the - // log line that says the gain is being taken over. - virtual QString description() const = 0; - - // driver is a name from AudioDevice::PlatformDriverName(), device the - // capture device string that goes with it. Returns null when this build - // or this platform has no way to reach the control, which is not an - // error: the caller carries on without automatic gain and the clipping - // warning still gets shown. - static CaptureGainControl *create(const QString &driver, const QString &device); -}; - -#endif // CAPTUREGAIN_H diff --git a/pica-client/capturegain_coreaudio.cpp b/pica-client/capturegain_coreaudio.cpp deleted file mode 100644 index 919d08d..0000000 --- a/pica-client/capturegain_coreaudio.cpp +++ /dev/null @@ -1,290 +0,0 @@ -/* - (c) Copyright 2012 - 2026 Anton Sviridenko - https://picapica.im - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, version 3. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. -*/ -#include "capturegain.h" - -#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) - -#include <QDebug> -#include <QVector> - -#include <CoreAudio/CoreAudio.h> -#include <CoreFoundation/CoreFoundation.h> - -// Small local copies of the device lookup helpers in -// nativeaudio_coreaudio.cpp. They are static over there and this is the only -// other place that needs them; duplicating three short functions beats -// exporting an internal API out of that file. -static QVector<AudioDeviceID> allInputCapableDevices() -{ - QVector<AudioDeviceID> devices; - - AudioObjectPropertyAddress addr; - addr.mSelector = kAudioHardwarePropertyDevices; - addr.mScope = kAudioObjectPropertyScopeGlobal; - addr.mElement = kAudioObjectPropertyElementMaster; - - UInt32 size = 0; - if (AudioObjectGetPropertyDataSize(kAudioObjectSystemObject, &addr, 0, nullptr, &size) != noErr) - return devices; - - devices.resize(size / sizeof(AudioDeviceID)); - if (devices.isEmpty()) - return devices; - - if (AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, 0, nullptr, &size, - devices.data()) != noErr) - devices.clear(); - - return devices; -} - -static QString deviceUidOf(AudioDeviceID dev) -{ - AudioObjectPropertyAddress addr; - addr.mSelector = kAudioDevicePropertyDeviceUID; - addr.mScope = kAudioObjectPropertyScopeGlobal; - addr.mElement = kAudioObjectPropertyElementMaster; - - CFStringRef uid = nullptr; - UInt32 size = sizeof(uid); - if (AudioObjectGetPropertyData(dev, &addr, 0, nullptr, &size, &uid) != noErr || !uid) - return QString(); - - char buf[512] = {0}; - QString result; - if (CFStringGetCString(uid, buf, sizeof(buf), kCFStringEncodingUTF8)) - result = QString::fromUtf8(buf); - - CFRelease(uid); - return result; -} - -static AudioDeviceID defaultInputDevice() -{ - AudioObjectPropertyAddress addr; - addr.mSelector = kAudioHardwarePropertyDefaultInputDevice; - addr.mScope = kAudioObjectPropertyScopeGlobal; - addr.mElement = kAudioObjectPropertyElementMaster; - - AudioDeviceID dev = kAudioObjectUnknown; - UInt32 size = sizeof(dev); - if (AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, 0, nullptr, &size, &dev) != noErr) - return kAudioObjectUnknown; - - return dev; -} - -// The input volume of a CoreAudio device. -// -// Which element carries it varies: some devices expose a master (element 0), -// some only per channel controls, and plenty of built in microphones expose -// none at all and are simply not adjustable. open() works that out once and -// remembers the answer; a device with nothing settable fails to open, and the -// caller falls back to warning the user. -class CoreAudioCaptureGain : public CaptureGainControl -{ -public: - CoreAudioCaptureGain() : m_dev(kAudioObjectUnknown), m_master(false) {} - - bool open(const QString &device) - { - if (device.isEmpty() || device == QLatin1String("default")) - { - m_dev = defaultInputDevice(); - } - else - { - QVector<AudioDeviceID> devices = allInputCapableDevices(); - for (int i = 0; i < devices.size(); i++) - { - if (deviceUidOf(devices[i]) == device) - { - m_dev = devices[i]; - break; - } - } - - // Fall back rather than give up - the same thing the audio unit - // does when a remembered device has been unplugged. - if (m_dev == kAudioObjectUnknown) - m_dev = defaultInputDevice(); - } - - if (m_dev == kAudioObjectUnknown) - return false; - - if (settable(0)) - { - m_master = true; - m_channels.clear(); - } - else - { - // Stereo inputs are the common case; anything past the second - // channel is rare enough on a microphone not to chase. - for (UInt32 ch = 1; ch <= 2; ch++) - { - if (settable(ch)) - m_channels << ch; - } - - if (m_channels.isEmpty()) - { - qWarning() << "CoreAudio: input device has no adjustable volume"; - return false; - } - } - - m_device = device.isEmpty() ? QStringLiteral("default") : device; - return true; - } - - bool gain(double *out) override - { - if (!out || m_dev == kAudioObjectUnknown) - return false; - - // The loudest of the channels, so stepping down moves all of them. - Float32 best = -1.0f; - - if (m_master) - { - if (!readElement(0, &best)) - return false; - } - else - { - for (int i = 0; i < m_channels.size(); i++) - { - Float32 v = 0.0f; - if (readElement(m_channels[i], &v) && v > best) - best = v; - } - } - - if (best < 0.0f) - return false; - - *out = best; - return true; - } - - // kAudioDevicePropertyVolumeDecibels is the same control as the scalar - // above, reported on a decibel scale. Devices that expose the scalar do not - // always expose this one, hence the plain false rather than a fallback. - bool gainDb(double *out) override - { - if (!out || m_dev == kAudioObjectUnknown) - return false; - - AudioObjectPropertyAddress addr; - addr.mSelector = kAudioDevicePropertyVolumeDecibels; - addr.mScope = kAudioObjectPropertyScopeInput; - addr.mElement = m_master ? 0 : (m_channels.isEmpty() ? 0 : m_channels[0]); - - if (!AudioObjectHasProperty(m_dev, &addr)) - return false; - - Float32 db = 0.0f; - UInt32 size = sizeof(db); - if (AudioObjectGetPropertyData(m_dev, &addr, 0, nullptr, &size, &db) != noErr) - return false; - - *out = db; - return true; - } - - bool setGain(double value) override - { - if (m_dev == kAudioObjectUnknown) - return false; - - Float32 v = (Float32)(value < 0.0 ? 0.0 : (value > 1.0 ? 1.0 : value)); - - if (m_master) - return writeElement(0, v); - - bool any = false; - for (int i = 0; i < m_channels.size(); i++) - { - if (writeElement(m_channels[i], v)) - any = true; - } - - return any; - } - - QString description() const override - { - return m_device + QStringLiteral(" (CoreAudio input volume)"); - } - -private: - static AudioObjectPropertyAddress addressFor(UInt32 element) - { - AudioObjectPropertyAddress addr; - addr.mSelector = kAudioDevicePropertyVolumeScalar; - addr.mScope = kAudioObjectPropertyScopeInput; - addr.mElement = element; - return addr; - } - - bool settable(UInt32 element) - { - AudioObjectPropertyAddress addr = addressFor(element); - - if (!AudioObjectHasProperty(m_dev, &addr)) - return false; - - Boolean writable = false; - if (AudioObjectIsPropertySettable(m_dev, &addr, &writable) != noErr) - return false; - - return writable; - } - - bool readElement(UInt32 element, Float32 *out) - { - AudioObjectPropertyAddress addr = addressFor(element); - UInt32 size = sizeof(*out); - return AudioObjectGetPropertyData(m_dev, &addr, 0, nullptr, &size, out) == noErr; - } - - bool writeElement(UInt32 element, Float32 value) - { - AudioObjectPropertyAddress addr = addressFor(element); - return AudioObjectSetPropertyData(m_dev, &addr, 0, nullptr, sizeof(value), &value) == noErr; - } - - AudioDeviceID m_dev; - bool m_master; - QVector<UInt32> m_channels; - QString m_device; -}; - -CaptureGainControl *pica_create_coreaudio_capture_gain(const QString &device) -{ - CoreAudioCaptureGain *g = new CoreAudioCaptureGain(); - - if (g->open(device)) - return g; - - delete g; - return nullptr; -} - -#endif // Q_OS_MACOS diff --git a/pica-client/capturegain_wasapi.cpp b/pica-client/capturegain_wasapi.cpp deleted file mode 100644 index 49ea84d..0000000 --- a/pica-client/capturegain_wasapi.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - (c) Copyright 2012 - 2026 Anton Sviridenko - https://picapica.im - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, version 3. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. -*/ -#include "capturegain.h" - -#ifdef Q_OS_WIN - -#include <QDebug> - -// Deliberately NOT defining INITGUID here, unlike nativeaudio_wasapi.cpp: that -// file already emits the definitions for the endpoint GUIDs, and a second copy -// in this translation unit would collide at link time. The interface IDs below -// come from __uuidof() instead, which mingw-w64 supports because its headers -// declare these interfaces with __CRT_UUID_DECL. -#include <windows.h> -#include <objbase.h> -#include <mmdeviceapi.h> -#include <endpointvolume.h> - -static QString hrText(HRESULT hr) -{ - return QString("0x%1").arg((quint32)hr, 8, 16, QLatin1Char('0')); -} - -// The endpoint volume, which is the slider under Settings > Sound > Input. -// -// Note this is not the same control as "Microphone Boost" - that one is a -// driver specific APO with no uniform API, and is usually the thing actually -// responsible when a mic clips at every level. Turning the endpoint volume -// down still reduces what reaches the converter on the great majority of -// devices, so it is the right control to reach for; it just may not have as -// much travel as the ALSA case. -class WasapiCaptureGain : public CaptureGainControl -{ -public: - WasapiCaptureGain() : m_vol(nullptr), m_uninitCom(false) {} - - ~WasapiCaptureGain() override - { - if (m_vol) - m_vol->Release(); - if (m_uninitCom) - CoUninitialize(); - } - - bool open(const QString &device) - { - HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); - // S_FALSE means this thread was already in an apartment, and - // RPC_E_CHANGED_MODE that it is in a different one - in both cases COM - // is usable and we must not be the one to tear it down. - if (SUCCEEDED(hr)) - m_uninitCom = true; - else if (hr != RPC_E_CHANGED_MODE) - return false; - - IMMDeviceEnumerator *devEnum = nullptr; - hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, - __uuidof(IMMDeviceEnumerator), (void **)&devEnum); - if (FAILED(hr) || !devEnum) - return false; - - IMMDevice *dev = nullptr; - if (device.isEmpty() || device == QLatin1String("default")) - { - // eCommunications to match what the capture stream itself opens in - // WasapiBackend::resolveDevice() - otherwise on a machine where the - // call endpoint and the console endpoint differ we would be turning - // down a device nobody is recording from. - hr = devEnum->GetDefaultAudioEndpoint(eCapture, eCommunications, &dev); - } - else - { - hr = devEnum->GetDevice((LPCWSTR)device.utf16(), &dev); - } - - devEnum->Release(); - - if (FAILED(hr) || !dev) - return false; - - hr = dev->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, nullptr, (void **)&m_vol); - dev->Release(); - - if (FAILED(hr) || !m_vol) - { - qWarning() << "WASAPI: capture endpoint has no volume control:" << hrText(hr); - m_vol = nullptr; - return false; - } - - m_device = device.isEmpty() ? QStringLiteral("default") : device; - return true; - } - - bool gain(double *out) override - { - if (!m_vol || !out) - return false; - - float level = 0.0f; - if (FAILED(m_vol->GetMasterVolumeLevelScalar(&level))) - return false; - - *out = level; - return true; - } - - // WASAPI keeps a real decibel value alongside the scalar, so this needs no - // conversion of our own. - bool gainDb(double *out) override - { - if (!m_vol || !out) - return false; - - float db = 0.0f; - if (FAILED(m_vol->GetMasterVolumeLevel(&db))) - return false; - - *out = db; - return true; - } - - bool setGain(double value) override - { - if (!m_vol) - return false; - - float level = (float)(value < 0.0 ? 0.0 : (value > 1.0 ? 1.0 : value)); - return SUCCEEDED(m_vol->SetMasterVolumeLevelScalar(level, nullptr)); - } - - QString description() const override - { - return m_device + QStringLiteral(" (WASAPI endpoint volume)"); - } - -private: - IAudioEndpointVolume *m_vol; - bool m_uninitCom; - QString m_device; -}; - -CaptureGainControl *pica_create_wasapi_capture_gain(const QString &device) -{ - WasapiCaptureGain *g = new WasapiCaptureGain(); - - if (g->open(device)) - return g; - - delete g; - return nullptr; -} - -#endif // Q_OS_WIN diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index a951b40..8685936 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -195,18 +195,38 @@ SettingsDialog::SettingsDialog(QWidget *parent) : audioTestStatus = new QLabel(this); audioTestStatus->setAlignment(Qt::AlignCenter); - cbEchoCancel = new QCheckBox(tr("Cancel acoustic echo"), this); - cbEchoCancel->setToolTip(tr("Removes the sound of the other person coming back out of your " - "microphone, which is what makes them hear themselves when you " - "are on speakers. Has no effect where the operating system " - "already does it.")); - - cbAutoCaptureGain = new QCheckBox(tr("Automatic microphone gain"), this); - cbAutoCaptureGain->setToolTip(tr("Turns the microphone's own recording level down during a " - "call if it is loud enough to distort, and puts it back " - "afterwards. A microphone set too loud clips, and clipping " - "cannot be repaired later - it also stops echo cancellation " - "working.")); + // Echo cancellation: removing the sound of the other person coming back + // out of the microphone, which is what makes them hear themselves when + // the call is on speakers. Exactly one of the three - two cancellers in + // series is worse than either alone, since the second one is left + // chasing a reference that is no longer in the signal. + QGroupBox *gbEchoCancel = new QGroupBox(tr("Audio echo cancellation"), this); + QVBoxLayout *echoCancelLayout = new QVBoxLayout(); + + // Parented to the group box, not the dialog: QRadioButton's automatic + // exclusivity groups by parent widget, and these three must not join the + // direct connection or multiple login buttons. + rbEchoCancelOwn = new QRadioButton(tr("Use own echo cancellation"), gbEchoCancel); + rbEchoCancelOwn->setToolTip(tr("Cancels the echo inside Pica Pica, using WebRTC's AEC3. " + "Works with any microphone and any sound system, and is the " + "only option where the platform provides nothing.")); + + rbEchoCancelPlatform = new QRadioButton(tr("Use platform provided echo cancellation"), gbEchoCancel); + + rbEchoCancelNone = new QRadioButton(tr("Disable echo cancellation"), gbEchoCancel); + rbEchoCancelNone->setToolTip(tr("Nothing is removed from the microphone signal. Right with a " + "headset, where there is no echo to cancel, and worth trying " + "if cancellation is cutting into your speech.")); + + echoCancelLayout->addWidget(rbEchoCancelOwn); + echoCancelLayout->addWidget(rbEchoCancelPlatform); + echoCancelLayout->addWidget(rbEchoCancelNone); + gbEchoCancel->setLayout(echoCancelLayout); + + // Fills in the platform option's tooltip and greys it out where there is + // no such thing. loadSettings() runs afterwards and is what actually + // picks one of the three. + updateEchoCancellationChoices(); btRingTest = new QPushButton(tr("Test ring 🔔"), this); connect(btRingTest, SIGNAL(clicked()), this, SLOT(toggleRingTest())); @@ -224,8 +244,7 @@ SettingsDialog::SettingsDialog(QWidget *parent) : audiodevlayout->addWidget(audioCaptureDev); audiodevlayout->addWidget(lbAudioPlaybackDev); audiodevlayout->addWidget(audioPlaybackDev); - audiodevlayout->addWidget(cbEchoCancel); - audiodevlayout->addWidget(cbAutoCaptureGain); + audiodevlayout->addWidget(gbEchoCancel); audiodevlayout->addWidget(btAudioTest); audiodevlayout->addWidget(audioTestStatus); audiodevlayout->addWidget(lbAudioRingDev); @@ -908,6 +927,43 @@ void SettingsDialog::audioDriverChanged(int index) fillAudioCaptureDevices(); fillAudioPlaybackDevices(); fillAudioRingDevices(); + + // ALSA has no echo cancellation of its own and PulseAudio does, so the + // platform option comes and goes with this choice. + updateEchoCancellationChoices(); +} + +void SettingsDialog::updateEchoCancellationChoices() +{ + // PlatformDriverName() reads the driver SetLinuxDriverName() published, + // which audioDriverChanged() has already updated by the time it calls + // this - so the answer follows the combo box rather than the saved + // setting. + const QString driver = AudioDevice::PlatformDriverName(CAPTURE); + const bool available = AudioDevice::PlatformEchoCancellationAvailable(driver); + + rbEchoCancelPlatform->setEnabled(available); + + if (available) + { + rbEchoCancelPlatform->setToolTip(tr("Lets the operating system or sound server cancel the " + "echo instead. Where it exists this is the better " + "one: it sits below Pica Pica and can see the real " + "speaker signal and the real clock.\n\n" + "With PulseAudio or PipeWire it also means picking an " + "echo cancelling source as the microphone above - a " + "plain input device is not processed.")); + } + else + { + rbEchoCancelPlatform->setToolTip(tr("Not available: the audio system in use does not " + "cancel echo for us.")); + + // A disabled button that is still the checked one would leave the + // group showing a choice that is not in effect. + if (rbEchoCancelPlatform->isChecked()) + rbEchoCancelOwn->setChecked(true); + } } void SettingsDialog::fillAudioCaptureDevices() @@ -1061,8 +1117,27 @@ void SettingsDialog::loadSettings() if (audioPlaybackDevItem >= 0) audioPlaybackDev->setCurrentIndex(audioPlaybackDevItem); - cbEchoCancel->setChecked(st.loadValue("audio.echo_cancel", 1).toBool()); - cbAutoCaptureGain->setChecked(st.loadValue("audio.auto_capture_gain", 0).toBool()); + // Not read from st directly: the setting has a default that depends on the + // platform, and an older on/off setting to fall back on. + switch (loadEchoCancellationSetting()) + { + case EchoCancellationPlatform: + rbEchoCancelPlatform->setChecked(true); + break; + + case EchoCancellationNone: + rbEchoCancelNone->setChecked(true); + break; + + case EchoCancellationOwn: + default: + rbEchoCancelOwn->setChecked(true); + break; + } + + // Covers a saved "platform" on a machine where it is no longer on offer - + // a Linux install whose audio system has since been moved to ALSA. + updateEchoCancellationChoices(); QString audioRingDevVal = st.loadValue("audio.ring_device", "default").toString(); int audioRingDevItem = audioRingDev->findData(audioRingDevVal); @@ -1116,8 +1191,6 @@ void SettingsDialog::storeSettings() st.storeValue("audio.capture_device", audioCaptureDev->itemData(audioCaptureDev->currentIndex()).toString()); st.storeValue("audio.playback_device", audioPlaybackDev->itemData(audioPlaybackDev->currentIndex()).toString()); st.storeValue("audio.ring_device", audioRingDev->itemData(audioRingDev->currentIndex()).toString()); - st.storeValue("audio.echo_cancel", cbEchoCancel->isChecked() ? "1" : "0"); - st.storeValue("audio.auto_capture_gain", cbAutoCaptureGain->isChecked() ? "1" : "0"); if (audioDriver) { @@ -1125,4 +1198,20 @@ void SettingsDialog::storeSettings() st.storeValue("audio.driver", drv); AudioDevice::SetLinuxDriverName(drv); } + + // After the driver, since on Linux that is what decides whether the + // platform option means anything - and updateEchoCancellationChoices() + // has already moved the selection off it if it does not. + EchoCancellationMode echoMode = EchoCancellationOwn; + + if (rbEchoCancelPlatform->isChecked()) + echoMode = EchoCancellationPlatform; + else if (rbEchoCancelNone->isChecked()) + echoMode = EchoCancellationNone; + + storeEchoCancellationSetting(echoMode); + + // Publish it to the audio threads, the same way the driver above is + // published - see AudioDevice::SetEchoCancellation(). + AudioDevice::SetEchoCancellation(echoMode); } diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h index b879a74..ba04612 100644 --- a/pica-client/dialogs/settingsdialog.h +++ b/pica-client/dialogs/settingsdialog.h @@ -78,10 +78,18 @@ private: // a slot hidden behind one ends up declared but never generated. QComboBox *audioDriver; - // Whether to run our own acoustic echo cancellation. Ignored when the - // platform provides one - see AudioDevice::PlatformDriverName(). - QCheckBox *cbEchoCancel; - QCheckBox *cbAutoCaptureGain; + // Who takes the echo of the other person back out of the microphone + // signal: EchoCanceller, the platform, or nobody. See + // EchoCancellationMode - one of the three, never two at once. + // + // The platform option is only offered where there is something to offer; + // on Linux that depends on the audio system chosen above it, so it is + // enabled and disabled as that combo box changes rather than once at + // startup. + QRadioButton *rbEchoCancelOwn; + QRadioButton *rbEchoCancelPlatform; + QRadioButton *rbEchoCancelNone; + void updateEchoCancellationChoices(); QComboBox *videoDev; QPushButton *videoDevRefresh; diff --git a/pica-client/echocanceller.cpp b/pica-client/echocanceller.cpp index 2d7acf0..ad5b122 100644 --- a/pica-client/echocanceller.cpp +++ b/pica-client/echocanceller.cpp @@ -15,15 +15,47 @@ along with this program. If not, see <https://www.gnu.org/licenses/>. */ #include "echocanceller.h" +#include "audiodevice.h" +#include "settings.h" +#include "globals.h" #include <QDebug> #include <QMutexLocker> #include <cmath> #include <cstring> +// Which major version of webrtc-audio-processing this is being built against. +// 1.x and 2.x are separate parallel-installable libraries with separate +// pkg-config names, and 2.x broke the API in a few places - the conditionals +// on this below are the whole of what differs as far as this file is +// concerned. +// +// configure defines it, and so does the qmake build. The fallback is for +// anything that builds this file without either: api/audio/audio_processing.h +// is where 2.x moved the header that 1.x only had under modules/, so its +// presence is the version. +#ifndef PICA_WEBRTC_AUDIO_PROCESSING_MAJOR +# if defined(__has_include) && __has_include(<api/audio/audio_processing.h>) +# define PICA_WEBRTC_AUDIO_PROCESSING_MAJOR 2 +# else +# define PICA_WEBRTC_AUDIO_PROCESSING_MAJOR 1 +# endif +#endif + +// Still the right include on 2.x, where it is a stub forwarding to +// api/audio/audio_processing.h - and it is installed there, so there is +// nothing to switch on here. +#include <modules/audio_processing/include/audio_processing.h> + +// Must come after the above, and not only as a matter of taste: on 2.x this +// header declares parameters as absl::Nullable/absl::Nonnull without +// including the abseil header that defines them, so on its own it does not +// compile. audio_processing.h is what pulls that in. Both versions include +// this themselves anyway; it is named here so that the scoped_refptr below is +// not silently relying on somebody else's include. +#include <api/scoped_refptr.h> + extern "C" { -#include <speex/speex_echo.h> -#include <speex/speex_preprocess.h> #include <libswresample/swresample.h> #include <libavutil/channel_layout.h> #include <libavutil/samplefmt.h> @@ -39,136 +71,217 @@ extern "C" { # define EC_HAVE_CH_LAYOUT 0 #endif -EchoCanceller::EchoCanceller(int sampleRate, int frameMs, int tailMs) - : m_sampleRate(sampleRate > 0 ? sampleRate : 48000), +// Setting values. Words rather than numbers because this is a three way +// choice that will be read by whoever is looking at a user's config, and a +// stray "2" says nothing. +static const char kEchoCancellationOwn[] = "own"; +static const char kEchoCancellationPlatform[] = "platform"; +static const char kEchoCancellationNone[] = "off"; + +EchoCancellationMode loadEchoCancellationSetting() +{ + Settings st(config_dbname); + + const QString value = st.loadValue("audio.echo_cancellation", QString()).toString(); + + if (value == QLatin1String(kEchoCancellationOwn)) + return EchoCancellationOwn; + + if (value == QLatin1String(kEchoCancellationPlatform)) + { + // The stored choice can stop being available underneath the user - + // the audio system moved from PulseAudio to ALSA, say. Leaving it as + // it stands would mean nobody cancelling and nothing saying so. + if (AudioDevice::PlatformEchoCancellationAvailable()) + return EchoCancellationPlatform; + + qWarning() << "Echo cancellation is set to the platform's own, but the audio system in " + "use does not provide one - falling back to ours"; + return EchoCancellationOwn; + } + + if (value == QLatin1String(kEchoCancellationNone)) + return EchoCancellationNone; + + // Nothing stored under the current key. An install that predates it may + // still have the on/off setting this replaced, and someone who turned + // cancellation off did mean it. + if (!st.loadValue("audio.echo_cancel", 1).toBool()) + return EchoCancellationNone; + + // The platform's own is the better one where it exists, so it is the + // default there; elsewhere the default is ours, which is what the plain + // on/off setting used to mean. + return AudioDevice::PlatformEchoCancellationAvailable() + ? EchoCancellationPlatform + : EchoCancellationOwn; +} + +void storeEchoCancellationSetting(EchoCancellationMode mode) +{ + Settings st(config_dbname); + + switch (mode) + { + case EchoCancellationPlatform: + st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationPlatform)); + break; + + case EchoCancellationNone: + st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationNone)); + break; + + case EchoCancellationOwn: + default: + st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationOwn)); + break; + } +} + +// Holds everything that needs the WebRTC headers, so that echocanceller.h - +// which audiodevice.h includes, and so half the client with it - does not. +struct EchoCancellerPrivate +{ + rtc::scoped_refptr<webrtc::AudioProcessing> apm; + + // Both directions are mono at our rate. The integer interface requires + // the capture, playback and render rates to be the same, which is why + // pushFarEnd() resamples rather than passing the peer's rate through. + webrtc::StreamConfig stream; +}; + +bool EchoCanceller::isNativeRate(int rate) +{ + return rate == 8000 || rate == 16000 || rate == 32000 || rate == 48000; +} + +EchoCanceller::EchoCanceller(int sampleRate) + : m_sampleRate(sampleRate), m_frameSize(0), - m_filterLength(0), - m_echo(nullptr), - m_preprocess(nullptr), - m_farHead(0), + m_d(nullptr), m_farSwr(nullptr), m_farSwrRate(0), - m_farCapacity(0), m_nearEnergy(0.0), - m_outEnergy(0.0), m_farEnergy(0.0), m_processedSamples(0), - m_nextReportSamples(0) + m_nextReportSamples(0), + m_renderBlocks(0), + m_captureBlocks(0) { - if (frameMs <= 0) - frameMs = 10; - if (tailMs < frameMs) - tailMs = frameMs; - - m_frameSize = (m_sampleRate * frameMs) / 1000; - m_filterLength = (m_sampleRate * tailMs) / 1000; - - // speexdsp's MDF works in blocks of frame_size and wants the filter to - // be a whole number of them. - if (m_frameSize <= 0) - m_frameSize = 480; - m_filterLength = ((m_filterLength + m_frameSize - 1) / m_frameSize) * m_frameSize; - - // Room for the sound card's output buffer plus a healthy margin. See the - // comment on m_farCapacity in the header for what overflowing it means. - m_farCapacity = m_sampleRate; // one second - - SpeexEchoState *echo = speex_echo_state_init(m_frameSize, m_filterLength); - if (!echo) + if (!isNativeRate(m_sampleRate)) { - qWarning() << "Could not create speex echo canceller state"; + qWarning() << "Echo canceller:" << m_sampleRate + << "Hz is not one of AEC3's rates (8/16/32/48 kHz) - not cancelling"; return; } - int rate = m_sampleRate; - speex_echo_ctl(echo, SPEEX_ECHO_SET_SAMPLING_RATE, &rate); + // AEC3 works on 10ms of audio at a time, everywhere, at every rate. + m_frameSize = m_sampleRate / 100; - SpeexPreprocessState *pre = speex_preprocess_state_init(m_frameSize, m_sampleRate); - if (pre) - { - // The adaptive filter removes the part of the echo it can model; what - // is left over - non-linearities in a cheap speaker, the tail of the - // room past m_filterLength - has to be suppressed statistically, and - // that is what the preprocessor's echo suppressor does with the - // residual estimate the echo state hands it. - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_STATE, echo); - - int suppress = -40; - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS, &suppress); - int suppressActive = -15; - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS_ACTIVE, &suppressActive); - - // Leave denoise on - it costs little next to the filter and keeps the - // residual suppressor's noise estimate honest. AGC stays off: the - // call has no level control anywhere else, and having one end quietly - // ride the gain makes the echo path non-stationary, which is the one - // thing an adaptive filter cannot follow. - int on = 1, off = 0; - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_DENOISE, &on); - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_AGC, &off); - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_VAD, &off); - } - else + webrtc::AudioProcessing::Config cfg; + + cfg.echo_canceller.enabled = true; + // AEC3 proper. The mobile mode is the cut down fixed point canceller for + // phones, and is worse anywhere that can afford not to use it. + cfg.echo_canceller.mobile_mode = false; + cfg.echo_canceller.enforce_high_pass_filtering = true; + + cfg.high_pass_filter.enabled = true; + + // Costs little next to the filter, and a lower noise floor is what lets + // the residual echo suppressor tell echo from room noise. + cfg.noise_suppression.enabled = true; + cfg.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + + // Both AGCs stay off. The call has no level control anywhere else, and a + // gain riding the microphone moves the echo path, which is the one thing + // an adaptive filter cannot follow. + cfg.gain_controller1.enabled = false; + cfg.gain_controller2.enabled = false; + + cfg.transient_suppression.enabled = false; + +#if PICA_WEBRTC_AUDIO_PROCESSING_MAJOR < 2 + // All three were dropped in 2.x. The first two only gate statistics we + // never read; the third is a second, statistical echo estimator whose + // output we do not use either. Switching them off on 1.x is therefore + // the same configuration 2.x gives us for nothing. + cfg.voice_detection.enabled = false; + cfg.level_estimation.enabled = false; + cfg.residual_echo_detector.enabled = false; +#endif + + m_d = new EchoCancellerPrivate; + m_d->stream = webrtc::StreamConfig(m_sampleRate, 1); + +#if PICA_WEBRTC_AUDIO_PROCESSING_MAJOR >= 2 + m_d->apm = webrtc::AudioProcessingBuilder().Create(); +#else + // 1.x hands back a raw pointer to an object whose reference count is + // still zero - the scoped_refptr is what takes it to one, and dropping + // the result on the floor would leak rather than free. + m_d->apm = rtc::scoped_refptr<webrtc::AudioProcessing>(webrtc::AudioProcessingBuilder().Create()); +#endif + + if (!m_d->apm) { - qWarning() << "Could not create speex preprocessor - running without residual echo suppression"; + qWarning() << "Could not create the WebRTC audio processing module"; + delete m_d; + m_d = nullptr; + return; } - m_echo = reinterpret_cast<SpeexEchoState_ *>(echo); - m_preprocess = reinterpret_cast<SpeexPreprocessState_ *>(pre); + m_d->apm->ApplyConfig(cfg); - m_farBlock.resize(m_frameSize); - m_farFifo.reserve(m_farCapacity); + // Mono at our rate in both directions, which the integer interface + // requires anyway. Spelled out through a ProcessingConfig rather than the + // shorter Initialize() overload taking channel layouts, because that + // overload and the ChannelLayout enum it needs are gone in 2.x. + webrtc::ProcessingConfig streams; + streams.input_stream() = m_d->stream; + streams.output_stream() = m_d->stream; + streams.reverse_input_stream() = m_d->stream; + streams.reverse_output_stream() = m_d->stream; + + const int err = m_d->apm->Initialize(streams); + if (err != webrtc::AudioProcessing::kNoError) + { + qWarning() << "Could not initialise the WebRTC audio processing module:" << err; + delete m_d; + m_d = nullptr; + return; + } // Hold off the first report until there is a window's worth of audio // behind the averages, otherwise it only says the call just started. m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec; - qDebug() << "Echo canceller:" << m_sampleRate << "Hz, frame" << m_frameSize - << "samples, tail" << m_filterLength << "samples (" - << (m_filterLength * 1000 / m_sampleRate) << "ms )"; + // The library version is in there because it is otherwise invisible which + // of the two parallel-installable ones got linked, and they behave + // differently enough to be worth knowing about from a log alone. + qDebug("Echo canceller: WebRTC AEC3 (webrtc-audio-processing-%d), %d Hz, frame %d samples", + PICA_WEBRTC_AUDIO_PROCESSING_MAJOR, m_sampleRate, m_frameSize); } EchoCanceller::~EchoCanceller() { - // Order matters: the preprocessor holds a pointer to the echo state. - if (m_preprocess) - speex_preprocess_state_destroy(reinterpret_cast<SpeexPreprocessState *>(m_preprocess)); - if (m_echo) - speex_echo_state_destroy(reinterpret_cast<SpeexEchoState *>(m_echo)); + delete m_d; + if (m_farSwr) swr_free(&m_farSwr); } -void EchoCanceller::reset() +bool EchoCanceller::isValid() const { - { - QMutexLocker locker(&m_procMutex); - if (m_echo) - speex_echo_state_reset(reinterpret_cast<SpeexEchoState *>(m_echo)); - } - - { - QMutexLocker locker(&m_farMutex); - m_farFifo.clear(); - m_farHead = 0; - } - - { - QMutexLocker locker(&m_statMutex); - m_nearEnergy = 0.0; - m_outEnergy = 0.0; - m_farEnergy = 0.0; - m_processedSamples = 0; - m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec; - } + return m_d != nullptr; } void EchoCanceller::pushFarEnd(const qint16 *pcm, int nsamples, int srcRate) { - if (!m_echo || !pcm || nsamples <= 0) + if (!m_d || !pcm || nsamples <= 0) return; - QMutexLocker locker(&m_farMutex); + QMutexLocker locker(&m_renderMutex); if (srcRate > 0 && srcRate != m_sampleRate) { @@ -226,97 +339,59 @@ void EchoCanceller::pushFarEnd(const qint16 *pcm, int nsamples, int srcRate) nsamples = got; } - // Reclaim the space already consumed rather than growing without bound; - // doing it only once the head has run past half the buffer keeps this - // from turning into a memmove on every call. - if (m_farHead > 0 && m_farHead >= m_farFifo.size() / 2) - { - m_farFifo.remove(0, m_farHead); - m_farHead = 0; - } + int base = m_farPending.size(); + m_farPending.resize(base + nsamples); + memcpy(m_farPending.data() + base, pcm, nsamples * sizeof(qint16)); - int oldSize = m_farFifo.size(); - m_farFifo.resize(oldSize + nsamples); - memcpy(m_farFifo.data() + oldSize, pcm, nsamples * sizeof(qint16)); + double farEnergy = 0.0; + int blocks = m_farPending.size() / m_frameSize; - int pending = m_farFifo.size() - m_farHead; - if (pending > m_farCapacity) + for (int b = 0; b < blocks; b++) { - m_farHead += pending - m_farCapacity; - if (m_farHead >= m_farFifo.size()) - { - m_farFifo.clear(); - m_farHead = 0; - } - } -} + qint16 *farBlock = m_farPending.data() + b * m_frameSize; -bool EchoCanceller::popFarEnd(qint16 *dst, int nsamples) -{ - QMutexLocker locker(&m_farMutex); - - int pending = m_farFifo.size() - m_farHead; - if (pending <= 0) - return false; + for (int i = 0; i < m_frameSize; i++) + farEnergy += double(farBlock[i]) * double(farBlock[i]); - int n = pending < nsamples ? pending : nsamples; - memcpy(dst, m_farFifo.constData() + m_farHead, n * sizeof(qint16)); - m_farHead += n; + // In place: the module is happy for source and destination to be the + // same buffer, and nothing downstream wants the processed render + // signal - handing it over is only how AEC3 learns what is about to + // come back through the microphone. + m_d->apm->ProcessReverseStream(farBlock, m_d->stream, m_d->stream, farBlock); + } - if (n < nsamples) - memset(dst + n, 0, (nsamples - n) * sizeof(qint16)); + if (blocks > 0) + { + m_farPending.remove(0, blocks * m_frameSize); - return true; + QMutexLocker statLocker(&m_statMutex); + m_farEnergy = m_farEnergy * 0.95 + (farEnergy / (blocks * m_frameSize)) * 0.05; + m_renderBlocks += blocks; + } } void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) { - if (!m_echo || !pcm || nsamples < m_frameSize) + if (!m_d || !pcm || nsamples < m_frameSize) return; - QMutexLocker locker(&m_procMutex); - - SpeexEchoState *echo = reinterpret_cast<SpeexEchoState *>(m_echo); - SpeexPreprocessState *pre = reinterpret_cast<SpeexPreprocessState *>(m_preprocess); - double nearEnergy = 0.0; - double outEnergy = 0.0; - double farEnergy = 0.0; - int blocks = nsamples / m_frameSize; - for (int b = 0; b < blocks; b++) - { - // Not called "near" - windef.h still defines that as an empty macro - // on the mingw-w64 builds. - qint16 *nearBlock = pcm + b * m_frameSize; - for (int i = 0; i < m_frameSize; i++) - nearEnergy += double(nearBlock[i]) * double(nearBlock[i]); + { + QMutexLocker locker(&m_captureMutex); - if (!popFarEnd(m_farBlock.data(), m_frameSize)) + for (int b = 0; b < blocks; b++) { - // Nothing is being played, so there is no echo to remove. Still - // run the frame through, otherwise the filter's idea of the far - // end signal silently runs ahead of the microphone. - memset(m_farBlock.data(), 0, m_frameSize * sizeof(qint16)); - } - - // Measured before cancellation - this is the reference the filter is - // about to be given, so it says whether there was anything coming out - // of the speaker for it to work on. - for (int i = 0; i < m_frameSize; i++) - farEnergy += double(m_farBlock[i]) * double(m_farBlock[i]); - - // speex_echo_cancellation() is happy to work in place on the near end - // buffer, which is what lets this sit directly in the capture path - // without another copy. - speex_echo_cancellation(echo, nearBlock, m_farBlock.constData(), nearBlock); + // Not called "near" - windef.h still defines that as an empty + // macro on the mingw-w64 builds. + qint16 *nearBlock = pcm + b * m_frameSize; - if (pre) - speex_preprocess_run(pre, nearBlock); + for (int i = 0; i < m_frameSize; i++) + nearEnergy += double(nearBlock[i]) * double(nearBlock[i]); - for (int i = 0; i < m_frameSize; i++) - outEnergy += double(nearBlock[i]) * double(nearBlock[i]); + m_d->apm->ProcessStream(nearBlock, m_d->stream, m_d->stream, nearBlock); + } } // At least one whole block, guaranteed by the nsamples check on entry. @@ -326,14 +401,13 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) { QMutexLocker statLocker(&m_statMutex); - // Exponential forgetting, so erle() reflects the last few seconds + // Exponential forgetting, so the levels reflect the last few seconds // rather than the whole call. Per sample, so a change in the capture - // path's block size does not move the levels. + // path's block size does not move them. m_nearEnergy = m_nearEnergy * 0.95 + (nearEnergy / used) * 0.05; - m_outEnergy = m_outEnergy * 0.95 + (outEnergy / used) * 0.05; - m_farEnergy = m_farEnergy * 0.95 + (farEnergy / used) * 0.05; m_processedSamples += used; + m_captureBlocks += blocks; if (m_processedSamples >= m_nextReportSamples) { m_nextReportSamples = m_processedSamples + qint64(m_sampleRate) * kReportIntervalSec; @@ -347,23 +421,73 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) // does not matter for a log line. if (report) { - int depth = farEndDepth(); + int delayMs = 0; + double erleDb = 0.0; + double erlDb = 0.0; + bool haveErl = false; + + { + QMutexLocker locker(&m_captureMutex); + webrtc::AudioProcessingStats stats = m_d->apm->GetStatistics(); + + if (stats.delay_ms.has_value()) + delayMs = *stats.delay_ms; + if (stats.echo_return_loss_enhancement.has_value()) + erleDb = *stats.echo_return_loss_enhancement; + if (stats.echo_return_loss.has_value()) + { + erlDb = *stats.echo_return_loss; + haveErl = true; + } + } + + qint64 lead = 0; + { + QMutexLocker statLocker(&m_statMutex); + lead = m_renderBlocks - m_captureBlocks; + } - qDebug("AEC: erle %+.1f dB mic %.1f dBFS far %.1f dBFS ref backlog %d ms%s", - erle(), nearEndLevelDb(), farEndLevelDb(), - (depth * 1000) / m_sampleRate, - depth >= m_farCapacity ? " [FIFO FULL - reference drifting]" : ""); + // erl says how strongly the speaker couples back into the microphone, + // which is whether there is anything worth cancelling at all. AEC3 + // reports it negated (echo_remover.cc: -10*log10(erl)), so the more + // negative it is the weaker the coupling - a headset reads well down, + // speakers in a small room read close to zero. + // + // erle is how much of that coupling the filter is actually removing. + // It sits at its 0 dB floor (Erle::min = 1.f) until the filter + // converges, so a call that never leaves +0.2 dB is a call where the + // linear filter never got anywhere. + // + // The delay is what AEC3 has aligned the two streams at. Zero is not + // a delay: block_processor.cc reports 0 when the estimator has no + // value, so it means the filter is running unaligned and the erle + // beside it cannot be anything but the floor. + QString delayText = delayMs > 0 + ? QString("echo delay %1 ms").arg(delayMs) + : QStringLiteral("NOT ALIGNED"); + + QString erlText = haveErl + ? QString("erl %1 dB").arg(erlDb, 0, 'f', 1) + : QStringLiteral("erl n/a"); + + qDebug("AEC: erle %+.1f dB %s mic %.1f dBFS far %.1f dBFS %s render lead %d ms", + erleDb, qPrintable(erlText), nearEndLevelDb(), farEndLevelDb(), + qPrintable(delayText), (int)(lead * 10)); // a block is 10ms, always } } double EchoCanceller::erle() { - QMutexLocker locker(&m_statMutex); - - if (m_nearEnergy <= 0.0 || m_outEnergy <= 0.0) + if (!m_d) return 0.0; - return 10.0 * log10(m_nearEnergy / m_outEnergy); + QMutexLocker locker(&m_captureMutex); + + webrtc::AudioProcessingStats stats = m_d->apm->GetStatistics(); + + return stats.echo_return_loss_enhancement.has_value() + ? *stats.echo_return_loss_enhancement + : 0.0; } // Mean square of 16 bit samples to dBFS, with a floor so digital silence @@ -390,11 +514,3 @@ double EchoCanceller::farEndLevelDb() QMutexLocker locker(&m_statMutex); return level_db(m_farEnergy); } - -int EchoCanceller::farEndDepth() -{ - QMutexLocker locker(&m_farMutex); - - int pending = m_farFifo.size() - m_farHead; - return pending > 0 ? pending : 0; -} diff --git a/pica-client/echocanceller.h b/pica-client/echocanceller.h index e59f8d7..faf50dd 100644 --- a/pica-client/echocanceller.h +++ b/pica-client/echocanceller.h @@ -22,60 +22,79 @@ #include <QVector> #include <QSharedPointer> -struct SpeexEchoState_; -struct SpeexPreprocessState_; +struct EchoCancellerPrivate; struct SwrContext; -// Acoustic echo cancellation for a call, using speexdsp's frequency domain -// adaptive filter (MDF). +// Where the acoustic echo of a call is taken out of the microphone signal. +// One of these three, never two at once: running a second canceller over an +// already cancelled signal gives it a near silent reference to chase and it +// chews holes in the speech. +enum EchoCancellationMode +{ + // EchoCanceller below - WebRTC's AEC3, in process. + EchoCancellationOwn, + + // Whatever the operating system or sound server does below us, which is + // better where it exists: it sees the real speaker signal and the real + // clock. Not offered everywhere - see + // AudioDevice::PlatformEchoCancellationAvailable(). + EchoCancellationPlatform, + + // Nothing cancels. Correct with a headset, and the only honest answer + // when both of the above make things worse. + EchoCancellationNone +}; + +// The "audio.echo_cancellation" setting. Both must be called from a thread +// that may touch the settings database - see AudioDevice::SetLinuxDriverName() +// for why the audio threads may not. +// +// Nothing stored yet means the default for this platform: the platform's own +// where there is one, ours otherwise. The setting this replaced +// ("audio.echo_cancel", a plain on/off) is still honoured when it is all +// there is, so an existing install that had it switched off stays that way. +EchoCancellationMode loadEchoCancellationSetting(); +void storeEchoCancellationSetting(EchoCancellationMode mode); + +// Acoustic echo cancellation for a call, using WebRTC's AEC3 through the +// audio processing module. // // One instance is shared by the two AudioDevice objects of a call - the // playback one feeds it the far end signal it is about to hand to the sound // card via pushFarEnd(), the capture one runs the microphone signal through -// processNearEnd() before encoding. Both run on their own thread, so every -// entry point here takes a lock. +// processNearEnd() before encoding. Both run on their own thread, which is +// what AEC3 expects: one render thread, one capture thread. // // Because AudioVideoCallController tears a call down without joining those // two threads (see stopAudioPipeline()), ownership is shared: hold this // through a QSharedPointer so the object outlives whichever thread leaves // last. // -// Alignment between the two signals is left to the FIFO: in the steady state -// the playback side pushes samples at the same rate the capture side pulls -// them, and the backlog that builds up in the FIFO settles at exactly the -// sound card's buffer depth - which is also how far ahead of the speaker the -// playback side is running. What remains is the acoustic flight time from -// speaker to microphone plus any capture side buffering, and that is what the -// filter tail is for. +// One per call, built when the call starts and dropped when it ends. There is +// nothing worth carrying over: the previous call's room response is not this +// call's, and the module is cheap enough to build. +// +// Nothing here tries to line the two signals up. AEC3 has its own delay +// estimator and a render buffer to hold the far end audio until the echo of +// it arrives, which is the whole reason it copes with an application that +// hands over playback audio some way ahead of the speaker actually making a +// sound - and with that lead changing when the sound card's buffer does. class EchoCanceller { public: - // frameMs is the block the filter works on and must divide the audio - // handed to processNearEnd(). - // - // tailMs is the longest echo delay the filter can model at all; anything - // arriving later is invisible to it and can only be dealt with - // statistically by the residual suppressor. The fixed part of that delay - // is small - the capture path plus a metre or two of flight, call it 30ms - // - so nearly all of the tail is budget for the room's reverberation. - // - // It is not free in either direction. Too short and late reflections never - // get modelled and are heard as residual echo; too long and both the - // convergence time and the steady state misadjustment grow with the tap - // count. 300ms at 16kHz is 4800 taps, which is still fewer than the 7200 - // this ran at when calls were 48kHz with a 150ms tail - so the room - // coverage doubles and the filter is still shorter than it used to be. - EchoCanceller(int sampleRate, int frameMs = 10, int tailMs = 300); + // sampleRate must be one of AEC3's native rates - 8000, 16000, 32000 or + // 48000. Anything else leaves isValid() false rather than silently + // processing at the wrong rate; the caller then runs without + // cancellation. + explicit EchoCanceller(int sampleRate); ~EchoCanceller(); - bool isValid() const { return m_echo != nullptr; } + bool isValid() const; int sampleRate() const { return m_sampleRate; } - int frameSize() const { return m_frameSize; } - // Forget the adapted filter and drop any buffered far end audio. Called - // when a call starts, since the previous call's room response is not - // worth keeping and a stale FIFO would start it off misaligned. - void reset(); + // The block AEC3 works on: 10ms, always. Audio handed to + // processNearEnd() should be a whole number of these. + int frameSize() const { return m_frameSize; } // Playback side: the audio about to be written to the output device. // @@ -83,9 +102,7 @@ public: // we capture at a rate of our choosing but play back at whatever the peer // announced, so a call with a build that negotiates a different rate ends // up with the two directions disagreeing. Passing 0 means "same as ours". - // Anything else is resampled on the way into the FIFO - without this the - // playback side used to drop the canceller entirely on a rate mismatch, - // leaving the capture side cancelling against silence. + // Anything else is resampled on the way in. void pushFarEnd(const qint16 *pcm, int nsamples, int srcRate = 0); // Capture side: removes the echo of what pushFarEnd() was given, in @@ -93,74 +110,57 @@ public: // partial trailing block is passed through untouched. void processNearEnd(qint16 *pcm, int nsamples); - // Rough measure of how well it is working, in dB - how much quieter the - // output is than the microphone input while the far end is active. Zero - // until enough far end audio has gone through to mean anything. + // Echo return loss enhancement in dB, as AEC3 measures it: how much + // quieter the echo is after cancellation than before. Zero until the + // filter has had far end audio to work on. // // Only meaningful while the far end is actually talking, which is what // farEndLevelDb() is for: with nothing coming out of the speaker there is // no echo to remove and the figure means nothing. double erle(); - // Recent signal levels in dBFS, averaged over the same window as erle(). + // Recent signal levels in dBFS, averaged over a few seconds. // nearEndLevelDb() is the microphone before cancellation - a floor-level // reading there means the capture path is dead and nothing else in the // numbers is worth reading. double nearEndLevelDb(); double farEndLevelDb(); - // Far end samples waiting to be paired with microphone audio. In the - // steady state this settles at the depth of the sound card's output - // buffer and stays there, because both are drained by the same clock. - // Climbing towards m_farCapacity means capture and playback are running - // off different clocks - separate cards - and the reference is drifting - // out of alignment with the echo it is supposed to explain. - int farEndDepth(); - private: Q_DISABLE_COPY(EchoCanceller) int m_sampleRate; int m_frameSize; - int m_filterLength; - // speexdsp state. The echo canceller proper, plus a preprocessor whose - // only job here is the residual echo suppression that the adaptive - // filter alone cannot do (SPEEX_PREPROCESS_SET_ECHO_STATE). - SpeexEchoState_ *m_echo; - SpeexPreprocessState_ *m_preprocess; + // The audio processing module and the stream configurations handed to it, + // kept out of this header so that the WebRTC headers - which want C++17 + // and drag in abseil - are only seen by echocanceller.cpp. + EchoCancellerPrivate *m_d; - // Guards the speex state, which both directions reach into: the capture - // thread through speex_echo_cancellation(), and reset() from whichever - // thread starts the call. - QMutex m_procMutex; + // Guards the render direction: the far end accumulator, the resampler, + // and ProcessReverseStream(). Only the playback thread takes it. + QMutex m_renderMutex; - // Far end FIFO, written by the playback thread and drained by the - // capture thread. - QMutex m_farMutex; - QVector<qint16> m_farFifo; - int m_farHead; + // Guards the capture direction: ProcessStream() and the statistics read + // alongside it. Taken by the capture thread, and by whoever calls the + // erle() accessor. + QMutex m_captureMutex; + + // Far end audio that did not fill a whole 10ms block, waiting for the + // samples that will complete it. + QVector<qint16> m_farPending; // Rate conversion for a far end that is not running at our rate, built on - // first use and rebuilt if the rate changes. Only the playback thread - // touches these, under m_farMutex. + // first use and rebuilt if the rate changes. SwrContext *m_farSwr; int m_farSwrRate; QVector<qint16> m_farScratch; - // Upper bound on the FIFO, in samples. Reached only if the two sound - // cards' clocks drift apart or playback outruns capture; past this the - // oldest audio is dropped, since keeping it would mean the reference - // signal lagging further and further behind the echo it is meant to - // explain. - int m_farCapacity; - - // Exponentially averaged mean square of the near end (microphone), the - // cancelled output and the far end reference, for erle() and the level - // readings. Per sample rather than per call, so they stay comparable - // whatever block size the capture path hands us. + // Exponentially averaged mean square of the near end (microphone) and of + // the far end reference, for the level readings. Per sample rather than + // per call, so they stay comparable whatever block size the capture path + // hands us. double m_nearEnergy; - double m_outEnergy; double m_farEnergy; QMutex m_statMutex; @@ -170,13 +170,21 @@ private: qint64 m_processedSamples; qint64 m_nextReportSamples; + // 10ms blocks handed to each direction. AEC3 lines the two streams up by + // counting API calls, not by any clock, so the gap between these is what + // its alignment is measured in. In a healthy call it settles at the depth + // of the sound card's output buffer and stays there; a gap that keeps + // growing or shrinking means capture and playback are running off + // different clocks and the alignment is being dragged out from under the + // filter. Guarded by m_statMutex. + qint64 m_renderBlocks; + qint64 m_captureBlocks; + // How often the canceller reports how it is doing, in seconds. static const int kReportIntervalSec = 2; - // Scratch far end block, reused by processNearEnd(). - QVector<qint16> m_farBlock; - - bool popFarEnd(qint16 *dst, int nsamples); + // True if rate is one of the rates AEC3's integer interface accepts. + static bool isNativeRate(int rate); }; typedef QSharedPointer<EchoCanceller> EchoCancellerPtr; diff --git a/pica-client/main.cpp b/pica-client/main.cpp index 330f487..3972ab8 100644 --- a/pica-client/main.cpp +++ b/pica-client/main.cpp @@ -449,14 +449,21 @@ static bool create_config_dir() QFile::setPermissions(config_dbname, QFile::ReadOwner | QFile::WriteOwner); - // Publish the audio driver choice to the audio threads while we are still - // the only thread and the settings database is reachable - see - // AudioDevice::SetLinuxDriverName() for why it cannot be read on demand. + // Publish the audio driver choice and the echo cancellation setting to the + // audio threads while we are still the only thread and the settings + // database is reachable - see AudioDevice::SetLinuxDriverName() for why + // they cannot be read on demand. + // + // The driver first: on Linux it is what decides whether the platform has + // any cancellation to offer, which loadEchoCancellationSetting() needs in + // order to work out the default. { Settings audiost(config_dbname); AudioDevice::SetLinuxDriverName(audiost.loadValue("audio.driver", "alsa").toString()); } + AudioDevice::SetEchoCancellation(loadEchoCancellationSetting()); + /*if (!QFile::exists(config_dir + QDir::separator() + PICA_CLIENT_DHPARAMFILE)) { OpenSSLTool osslt; diff --git a/pica-client/nativeaudio.h b/pica-client/nativeaudio.h index fa3cccb..3e364c6 100644 --- a/pica-client/nativeaudio.h +++ b/pica-client/nativeaudio.h @@ -43,14 +43,22 @@ // What a stream is going to be used for. // -// A call stream asks the platform for its voice processing, echo cancellation -// above all. A plain one does not, and must not: on macOS the echo canceller -// is an audio unit that captures as well as renders, so asking for it to play -// a ringtone would open the microphone, prompt for permission and light the -// recording indicator, all to make a telephone noise. +// The two call modes differ in one thing only: whether the platform is asked +// for its voice processing, echo cancellation above all. Which of them a call +// uses is the "audio.echo_cancellation" setting - see EchoCancellationMode. +// They agree on everything else, device selection included: a call belongs on +// the endpoint the user nominated for calls whoever ends up cancelling the +// echo. +// +// A plain stream is not a call at all, and must not ask for voice processing: +// on macOS the echo canceller is an audio unit that captures as well as +// renders, so asking for it to play a ringtone would open the microphone, +// prompt for permission and light the recording indicator, all to make a +// telephone noise. enum NativeAudioMode { NativeAudioVoiceCall, + NativeAudioVoiceCallRaw, NativeAudioPlain }; diff --git a/pica-client/nativeaudio_coreaudio.cpp b/pica-client/nativeaudio_coreaudio.cpp index be0d604..6dc9ad7 100644 --- a/pica-client/nativeaudio_coreaudio.cpp +++ b/pica-client/nativeaudio_coreaudio.cpp @@ -147,6 +147,30 @@ static QVector<AudioDeviceID> allDevices() return devices; } +// The system's current default input or output device. +// +// Needed because kAudioUnitSubType_HALOutput left alone follows the default +// OUTPUT device on both of its buses, so an input only unit that is never +// bound to a device tries to capture from the speakers. VoiceProcessingIO has +// no such problem, which is why nothing needed this until there was a plain +// capture path. +static AudioDeviceID defaultDevice(bool input) +{ + AudioObjectPropertyAddress addr; + addr.mSelector = input ? kAudioHardwarePropertyDefaultInputDevice + : kAudioHardwarePropertyDefaultOutputDevice; + addr.mScope = kAudioObjectPropertyScopeGlobal; + addr.mElement = kAudioObjectPropertyElementMaster; + + AudioDeviceID dev = kAudioObjectUnknown; + UInt32 size = sizeof(dev); + + if (AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, 0, nullptr, &size, &dev) != noErr) + return kAudioObjectUnknown; + + return dev; +} + // Looks up a device by the UID string enumerate() handed out. static AudioDeviceID deviceForUid(const QString &uid, bool input) { @@ -174,10 +198,12 @@ static AudioDeviceID deviceForUid(const QString &uid, bool input) // through the same unit, and AudioDevice keeps one object per direction on its // own thread, the voice unit is a refcounted singleton the two of them share. // -// A plain unit is kAudioUnitSubType_HALOutput, output only, no voice -// processing, one per caller. Ringtones and earpiece tones use it: they are -// not part of a call, and opening a voice unit to play one would switch the -// microphone on, prompt for permission and light the recording indicator. +// A plain unit is kAudioUnitSubType_HALOutput with no voice processing, one +// per caller, running one direction only. Ringtones and earpiece tones use +// the output flavour: they are not part of a call, and opening a voice unit +// to play one would switch the microphone on, prompt for permission and light +// the recording indicator. The input flavour is for a call that cancels its +// own echo, or none of it - see NativeAudioVoiceCallRaw. // // One consequence of the singleton worth knowing about: an audio unit is // bound to one device, so capture and playback cannot be on different sound @@ -196,6 +222,12 @@ public: static CoreAudioUnit *createPlain(const QString &device, int sampleRate, int channels, QString *error); + // The same, input only: a microphone opened without the voice unit, for a + // call whose echo is cancelled somewhere else - by EchoCanceller, or not + // at all. Owned by the caller like createPlain(). + static CoreAudioUnit *createPlainCapture(const QString &device, int sampleRate, int channels, + QString *error); + void release(bool forCapture); AudioRing captureRing; @@ -312,25 +344,41 @@ bool CoreAudioUnit::setUp(const QString &device, QString *error) // Bind to a specific device when one was asked for. Left alone, the unit // follows the system default input and output, which is what most users // want and what keeps working when they unplug a headset mid-call. + // + // With one exception: a HAL unit follows the default OUTPUT device on + // both of its buses, so an input only one has to be pointed at the + // default input by hand or it captures from nothing. + AudioDeviceID devId = kAudioObjectUnknown; + if (!device.isEmpty() && device != QLatin1String("default")) { - AudioDeviceID devId = deviceForUid(device, m_wantInput); + devId = deviceForUid(device, m_wantInput); if (devId == kAudioObjectUnknown) devId = deviceForUid(device, !m_wantInput); - if (devId != kAudioObjectUnknown) - { - err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_CurrentDevice, - kAudioUnitScope_Global, kOutputBus, &devId, sizeof(devId)); - if (err != noErr) - qWarning() << "CoreAudio: could not bind the audio unit to device" << device - << osStatusString(err) << "- using the system default"; - } - else - { + if (devId == kAudioObjectUnknown) qWarning() << "CoreAudio: device" << device << "not found - using the system default"; + } + + if (devId == kAudioObjectUnknown && m_wantInput && !m_wantOutput) + { + devId = defaultDevice(true); + if (devId == kAudioObjectUnknown) + { + if (error) + *error = QStringLiteral("There is no default audio input device"); + return false; } } + + if (devId != kAudioObjectUnknown) + { + err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, kOutputBus, &devId, sizeof(devId)); + if (err != noErr) + qWarning() << "CoreAudio: could not bind the audio unit to device" << device + << osStatusString(err) << "- using the system default"; + } m_device = device; UInt32 maxFrames = kMaxFramesPerSlice; @@ -390,7 +438,7 @@ bool CoreAudioUnit::setUp(const QString &device, QString *error) kAudioUnitScope_Global, 0, &bypass, sizeof(bypass)); // Leave the unit's automatic gain control off, for the same reason - // the speexdsp path leaves speex's off: a moving gain in front of a + // EchoCanceller leaves WebRTC's off: a moving gain in front of a // call that has no level control anywhere else is more trouble than // it is worth. UInt32 agc = 0; @@ -616,6 +664,20 @@ CoreAudioUnit *CoreAudioUnit::createPlain(const QString &device, int sampleRate, return unit; } +CoreAudioUnit *CoreAudioUnit::createPlainCapture(const QString &device, int sampleRate, int channels, + QString *error) +{ + CoreAudioUnit *unit = new CoreAudioUnit(sampleRate, channels, false, true, false); + if (!unit->setUp(device, error)) + { + delete unit; + return nullptr; + } + + unit->m_captureRefs = 1; + return unit; +} + void CoreAudioUnit::release(bool forCapture) { QMutexLocker locker(&s_mutex); @@ -668,15 +730,18 @@ public: { close(); - // There is no plain capture path: the only reason this backend - // captures at all is the call, and the call always wants the voice - // unit. - Q_UNUSED(mode) + // The voice unit is the whole point of this backend, but it is not + // the only thing it can open: a call that cancels its own echo wants + // the microphone as the hardware delivers it, with nothing having + // been subtracted from it first. + if (mode == NativeAudioVoiceCall) + m_unit = CoreAudioUnit::acquireVoice(device, sampleRate, channels, true, error); + else + m_unit = CoreAudioUnit::createPlainCapture(device, sampleRate, channels, error); - m_unit = CoreAudioUnit::acquireVoice(device, sampleRate, channels, true, error); m_isCapture = true; m_channels = channels; - m_platformAec = (m_unit != nullptr); + m_platformAec = (m_unit != nullptr && m_unit->voiceProcessing()); return m_unit != nullptr; } diff --git a/pica-client/nativeaudio_wasapi.cpp b/pica-client/nativeaudio_wasapi.cpp index 449dcd7..c0f5130 100644 --- a/pica-client/nativeaudio_wasapi.cpp +++ b/pica-client/nativeaudio_wasapi.cpp @@ -299,7 +299,11 @@ IMMDevice *WasapiBackend::resolveDevice(const QString &device, bool isCapture, // A ringtone is not a call and belongs on the console endpoint, so // that it is heard on the speakers rather than in a headset the user // is not wearing yet. - const ERole role = (mode == NativeAudioVoiceCall) ? eCommunications : eConsole; + // + // Both call modes take the communications endpoint. Cancelling the + // echo ourselves is a reason not to ask for the endpoint's voice + // processing, not a reason to move the call to a different device. + const ERole role = (mode != NativeAudioPlain) ? eCommunications : eConsole; hr = devEnum->GetDefaultAudioEndpoint(isCapture ? eCapture : eRender, role, &dev); } else @@ -407,6 +411,10 @@ bool WasapiBackend::open(const QString &device, int sampleRate, int channels, bo // Asking for it on render as well is what made the settings dialog's // microphone test come out chopped while the ring test, which opens a // plain stream, was clean. + // + // NativeAudioVoiceCallRaw is a call that cancels its own echo, so it does + // not want the endpoint's processing either - this is the one place where + // it differs from NativeAudioVoiceCall. if (mode == NativeAudioVoiceCall && isCapture) requestCommunicationsCategory(); diff --git a/pica-client/pica-client.pro b/pica-client/pica-client.pro index 9dcc1ab..ddbcda8 100644 --- a/pica-client/pica-client.pro +++ b/pica-client/pica-client.pro @@ -16,9 +16,6 @@ SOURCES += main.cpp\ nativeaudio.cpp \ nativeaudio_wasapi.cpp \ nativeaudio_coreaudio.cpp \ - capturegain.cpp \ - capturegain_wasapi.cpp \ - capturegain_coreaudio.cpp \ audiovideocallcontroller.cpp \ mainwindow.cpp \ chatwindow.cpp \ @@ -62,7 +59,6 @@ HEADERS += mainwindow.h \ audioring.h \ echocanceller.h \ nativeaudio.h \ - capturegain.h \ audiovideocallcontroller.h \ chatwindow.h \ callwindow.h \ @@ -119,11 +115,34 @@ unix: PKGCONFIG += libcrypto unix|win32: LIBS += -lminiupnpc -# Acoustic echo cancellation. Required everywhere: on Linux there is no other -# cancellation in the picture, and on the platforms that have their own it is -# still the fallback for the endpoints and drivers that turn out not to. -unix:!macx: CONFIG += link_pkgconfig -unix:!macx: PKGCONFIG += speexdsp +# Acoustic echo cancellation - WebRTC's AEC3, through the audio processing +# module. Required everywhere: it is what the "use own echo cancellation" +# setting selects, and on Linux it is usually the only cancellation in the +# picture at all. +# +# Its headers, and the abseil ones they include, refuse to compile as anything +# older than C++17, so the whole target is built that way. +CONFIG += c++17 + +# Through pkg-config on macOS too, unlike the libraries above: the headers sit +# in a versioned include directory and the library needs several abseil ones +# alongside it, so the flags are not worth writing out by hand. +# +# 1.x and 2.x install in parallel under separate pkg-config names and differ +# in a few API details that echocanceller.cpp handles; 1.x wins when both are +# there, matching what configure does. The macro is only a hint - the file +# falls back to working the version out from the headers - but a build that +# has both installed is exactly where the hint earns its keep. +unix: CONFIG += link_pkgconfig +unix { + packagesExist(webrtc-audio-processing-1) { + PKGCONFIG += webrtc-audio-processing-1 + DEFINES += PICA_WEBRTC_AUDIO_PROCESSING_MAJOR=1 + } else { + PKGCONFIG += webrtc-audio-processing-2 + DEFINES += PICA_WEBRTC_AUDIO_PROCESSING_MAJOR=2 + } +} # Listing PulseAudio sources and sinks in the settings dialog. Optional - # playing and capturing through PulseAudio goes via FFmpeg's own "pulse" @@ -136,4 +155,3 @@ unix:!macx: DEFINES += HAVE_LIBPULSE # what they link against cannot. win32: LIBS += -lole32 -loleaut32 macx: LIBS += -framework CoreAudio -framework AudioUnit -framework AudioToolbox -framework CoreFoundation -macx: LIBS += -lspeexdsp |
