summaryrefslogtreecommitdiffstats
path: root/pica-client
diff options
context:
space:
mode:
authorAnton <anton@picapica.im>2026-09-13 19:53:28 +0400
committerAnton <anton@picapica.im>2026-09-13 19:53:28 +0400
commit9a84c95af6771590d4e5fe436e77bbe92a9dea92 (patch)
tree71511a89b296d02d4106f9f031de806c8db422b7 /pica-client
parente62dd7d30bec7a679013dcce8ec4bc2b2f842109 (diff)
downloadpica-pica-9a84c95af6771590d4e5fe436e77bbe92a9dea92.tar.gz
pica-pica-9a84c95af6771590d4e5fe436e77bbe92a9dea92.zip
pica-client: use webrtc-audio-processing instead of speexdsp for echo cancellation
Diffstat (limited to 'pica-client')
-rw-r--r--pica-client/audiodevice.cpp62
-rw-r--r--pica-client/audiodevice.h20
-rw-r--r--pica-client/audiovideocallcontroller.cpp255
-rw-r--r--pica-client/audiovideocallcontroller.h79
-rw-r--r--pica-client/capturegain.cpp533
-rw-r--r--pica-client/capturegain.h81
-rw-r--r--pica-client/capturegain_coreaudio.cpp290
-rw-r--r--pica-client/capturegain_wasapi.cpp168
-rw-r--r--pica-client/dialogs/settingsdialog.cpp125
-rw-r--r--pica-client/dialogs/settingsdialog.h16
-rw-r--r--pica-client/echocanceller.cpp492
-rw-r--r--pica-client/echocanceller.h174
-rw-r--r--pica-client/main.cpp13
-rw-r--r--pica-client/nativeaudio.h18
-rw-r--r--pica-client/nativeaudio_coreaudio.cpp109
-rw-r--r--pica-client/nativeaudio_wasapi.cpp10
-rw-r--r--pica-client/pica-client.pro38
17 files changed, 761 insertions, 1722 deletions
diff --git a/pica-client/audiodevice.cpp b/pica-client/audiodevice.cpp
index e04aca9..fdbeedc 100644
--- a/pica-client/audiodevice.cpp
+++ b/pica-client/audiodevice.cpp
@@ -80,6 +80,13 @@ static const int64_t kOpusBitrate = 32000;
static QMutex s_driverMutex;
static QString s_linuxDriver = QStringLiteral("alsa");
+// The configured echo cancellation, published to the audio threads the same
+// way and for the same reason. The initial value only stands until main()
+// has read the setting, and is the one that cannot make things worse if it
+// somehow does not.
+static QMutex s_echoCancellationMutex;
+static EchoCancellationMode s_echoCancellation = EchoCancellationNone;
+
static QString ff_errstr(int err)
{
char buf[256] = {0};
@@ -612,8 +619,16 @@ void AudioDevice::captureNative(const QString &api)
return;
}
+ // Ask the platform for its voice processing only when it is the one
+ // meant to be cancelling. Either of the other two settings wants the
+ // microphone as the hardware delivers it: EchoCanceller cannot model an
+ // echo path that something below it is already subtracting from, and
+ // "off" means off.
+ const NativeAudioMode mode = (AudioDevice::EchoCancellation() == EchoCancellationPlatform)
+ ? NativeAudioVoiceCall : NativeAudioVoiceCallRaw;
+
QString err;
- if (!backend->openCapture(m_deviceName, m_sampleRate, kChannels, NativeAudioVoiceCall, &err))
+ if (!backend->openCapture(m_deviceName, m_sampleRate, kChannels, mode, &err))
{
qWarning() << err;
emit errorOccurred(err);
@@ -1021,8 +1036,15 @@ void AudioDevice::playNative(const QString &api)
return;
}
+ // Same choice as captureNative(), and it has to be the same answer: on
+ // macOS the platform canceller is one audio unit doing both directions,
+ // so asking for it here and not there would open a second unit on the
+ // same device.
+ const NativeAudioMode mode = (AudioDevice::EchoCancellation() == EchoCancellationPlatform)
+ ? NativeAudioVoiceCall : NativeAudioVoiceCallRaw;
+
QString err;
- if (!backend->openPlayback(m_deviceName, m_sampleRate, kChannels, NativeAudioVoiceCall, &err))
+ if (!backend->openPlayback(m_deviceName, m_sampleRate, kChannels, mode, &err))
{
qWarning() << err;
emit errorOccurred(err);
@@ -1411,6 +1433,42 @@ QString AudioDevice::LinuxDriverName()
return s_linuxDriver;
}
+void AudioDevice::SetEchoCancellation(EchoCancellationMode mode)
+{
+ QMutexLocker locker(&s_echoCancellationMutex);
+ s_echoCancellation = mode;
+}
+
+EchoCancellationMode AudioDevice::EchoCancellation()
+{
+ QMutexLocker locker(&s_echoCancellationMutex);
+ return s_echoCancellation;
+}
+
+bool AudioDevice::PlatformEchoCancellationAvailable(const QString &driver)
+{
+ // Windows and macOS both have one, and both are reached through the
+ // native backends rather than through libavdevice - which is why those
+ // backends exist at all.
+ if (AudioDevice::IsPlatformDriver(driver))
+ return NativeAudioBackend::isAvailable(AudioDevice::PlatformApiName(driver));
+
+ // On Linux nothing is provided per stream. What there is instead is a
+ // sound server that can be made to hand out an already cancelled source -
+ // PulseAudio's module-echo-cancel, or PipeWire's equivalent - which the
+ // user then picks from the microphone list like any other device. Raw
+ // ALSA has nothing of the sort.
+ if (driver == QLatin1String("pulse"))
+ return true;
+
+ return false;
+}
+
+bool AudioDevice::PlatformEchoCancellationAvailable()
+{
+ return PlatformEchoCancellationAvailable(AudioDevice::PlatformDriverName(CAPTURE));
+}
+
QString AudioDevice::PlatformDriverName(enum MediaDeviceStreamDirection dir)
{
Q_UNUSED(dir)
diff --git a/pica-client/audiodevice.h b/pica-client/audiodevice.h
index 6b2db1c..b98acf2 100644
--- a/pica-client/audiodevice.h
+++ b/pica-client/audiodevice.h
@@ -55,7 +55,7 @@ public:
// far end signal; both directions of one call must be given the same
// object or there is nothing to cancel against.
//
- // Ignored when the platform is doing its own cancellation - see
+ // Null unless EchoCancellation() is EchoCancellationOwn - see
// AudioVideoCallController::startAudioPipeline(), which is where that is
// decided.
void setEchoCanceller(EchoCancellerPtr ec);
@@ -151,6 +151,24 @@ public:
static void SetLinuxDriverName(const QString &driver);
static QString LinuxDriverName();
+ // Whether anything below us would cancel the acoustic echo, so that
+ // EchoCancellationPlatform is worth offering at all.
+ //
+ // The overload taking a driver answers for that driver rather than the
+ // configured one, which is what the settings dialog needs: on Linux the
+ // answer depends on the audio system, and the radio button has to follow
+ // the combo box before either has been applied.
+ static bool PlatformEchoCancellationAvailable(const QString &driver);
+ static bool PlatformEchoCancellationAvailable();
+
+ // Which of the three cancellers a call should use. Cached in a process
+ // wide variable for the same reason LinuxDriverName() is - the capture
+ // and playback loops run on their own threads and cannot reach the
+ // settings database. Set it from the GUI thread at startup and whenever
+ // the setting changes.
+ static void SetEchoCancellation(EchoCancellationMode mode);
+ static EchoCancellationMode EchoCancellation();
+
private:
QString m_deviceName;
QString m_codec;
diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp
index 405746d..72b6fc2 100644
--- a/pica-client/audiovideocallcontroller.cpp
+++ b/pica-client/audiovideocallcontroller.cpp
@@ -30,12 +30,10 @@ static const quint16 kVideoHeight = 480;
// Rate we capture and encode at, announced to the peer in the 0x74 message.
//
-// 16kHz rather than 48kHz because of the echo canceller. speexdsp's MDF is
-// tuned for 8/16kHz, and its cost is counted in taps: a 150ms tail is 7200
-// taps at 48kHz but only 2400 at 16kHz. Convergence time and steady state
-// misadjustment both scale with that length, so the 48kHz filter spent every
-// call chasing and never settling - visible in the AEC log as ERLE climbing
-// towards 20dB and being knocked back to zero by each doubletalk burst.
+// 16kHz rather than 48kHz because it is what the echo canceller wants. AEC3
+// does its filtering in the 0-8kHz band whatever it is handed; at 48kHz it
+// splits the signal into four bands, cancels in the lowest and gates the rest,
+// which is more work for a result that is no better on speech.
//
// Costs nothing worth having for speech: Opus codes this as wideband, an 8kHz
// audio bandwidth, which is well past telephone quality and past where voice
@@ -49,10 +47,7 @@ static const int kCallSampleRate = 16000;
AudioVideoCallController::AudioVideoCallController(QObject *parent)
: QObject(parent), callwindow(0), ringdevice(0), is_active(false),
- m_captureGain(nullptr), m_savedCaptureGain(0.0), m_haveSavedCaptureGain(false),
- m_gainCalibrating(false), m_captureGainSteps(0), m_gainStartDb(0.0),
- m_haveGainDb(false), m_gainIneffectiveSteps(0), m_lastPinnedPercent(0.0),
- m_gainSettleReports(0), m_audioSeq(0),
+ m_audioSeq(0),
m_videoSeq(0), m_videoFrameTimestamp(0), m_videoFrameStarted(false)
{
connect(skynet, SIGNAL(IncomingCall(QByteArray)), this, SLOT(call_from(QByteArray)));
@@ -189,11 +184,11 @@ void AudioVideoCallController::startAudioPipeline()
// stopAudioPipeline() does not join either thread, so this has to outlive
// whichever of them returns last.
//
- // It is created even on the platforms that cancel for us, because whether
- // they really will is not known until the capture stream has been opened
- // - see AudioDevice::captureNative(), which drops it once the backend
- // says the platform has the job in hand.
- if (st.loadValue("audio.echo_cancel", 1).toBool())
+ // Only for EchoCancellationOwn. The other two settings mean something
+ // below us is doing the job or nobody is, and either way running this as
+ // well would only take a second bite out of the speech - the devices are
+ // opened accordingly, see AudioDevice::captureNative().
+ if (AudioDevice::EchoCancellation() == EchoCancellationOwn)
m_echoCanceller = EchoCancellerPtr(new EchoCanceller(kCallSampleRate));
else
m_echoCanceller.clear();
@@ -206,10 +201,6 @@ void AudioVideoCallController::startAudioPipeline()
microphone->setEchoCanceller(m_echoCanceller);
output->setEchoCanceller(m_echoCanceller);
- // Before Capture() starts, so the gain is already read and saved by the
- // time the first clipping report can come back.
- startCaptureGainControl(capDev);
-
// Declare our outgoing codec to the peer, then start capturing and
// encoding right away - the peer's decoder is only ready once it has
// processed this 0x74 message, but since the call is carried over TCP,
@@ -238,12 +229,6 @@ void AudioVideoCallController::stopAudioPipeline()
microphone->Close();
output->Close();
- // Puts the user's gain back. Safe to do while the capture thread is still
- // winding down: the control is only ever touched from this thread, and
- // capture_clipping() finds m_captureGain null and does nothing if a last
- // queued report arrives after this.
- stopCaptureGainControl();
-
// Drop our reference. The two audio threads still hold theirs until they
// come out of Capture()/Play(), and the canceller goes away with the last
// of them.
@@ -469,222 +454,20 @@ void AudioVideoCallController::send_audio_packet(QByteArray data)
}
// Rate limited by AudioDevice, which reports at most once every couple of
-// seconds of audio, so this can log and act unconditionally.
+// seconds of audio, so this can log unconditionally.
+//
+// Only a warning, and deliberately: clipping happens in the converter, before
+// anything here sees a sample, so nothing in this process can undo it. The
+// fix is less gain ahead of the ADC, which is the capture device's own mixer
+// control and the user's to set.
void AudioVideoCallController::capture_clipping(double pinnedPercent, double peakDb)
{
- qWarning("Microphone is clipping: %.2f%% of samples at full scale, peak %.1f dBFS",
+ qWarning("Microphone is clipping: %.2f%% of samples at full scale, peak %.1f dBFS - "
+ "turn the recording level down, it distorts the speech and stops the echo "
+ "canceller converging",
pinnedPercent, peakDb);
-
- // Either the setting is off, or nothing on this platform would let us at
- // the control. The warning above is then all we can do - and it is still
- // worth having, since the distortion cannot be undone anywhere else.
- if (!m_captureGain || !m_gainCalibrating)
- return;
-
- // Calibration is a thing that happens at the start of a call, not a gain
- // rider. Once the window has passed the level is whatever it is: moving it
- // mid-conversation would keep dragging the echo path out from under the
- // canceller, which is worse than a bit of distortion.
- if (m_callClock.elapsed() > kGainCalibrationMs)
- {
- closeGainCalibration("calibration window has passed");
- return;
- }
-
- // The audio in this report may predate the last adjustment.
- if (m_gainSettleReports > 0)
- {
- m_gainSettleReports--;
- return;
- }
-
- // Without a dB scale there is no safe way to know how far down we have
- // come, so the step count has to be the bound - and a much tighter one,
- // since six steps of 0.7 is over 50 dB on a perceptual scale.
- const int stepLimit = m_haveGainDb ? kMaxCaptureGainSteps : kMaxBlindGainSteps;
- if (m_captureGainSteps >= stepLimit)
- {
- closeGainCalibration("step limit reached");
- return;
- }
-
- // Did the previous adjustment achieve anything? Two that did not means the
- // stage overloading is not the one we are holding.
- if (m_captureGainSteps > 0 && pinnedPercent >= m_lastPinnedPercent)
- {
- m_gainIneffectiveSteps++;
- if (m_gainIneffectiveSteps >= kMaxGainIneffectiveSteps)
- {
- closeGainCalibration("lowering this control is not reducing the clipping - "
- "something ahead of it (an analog mic boost, or Windows' "
- "separate Microphone Boost) is what is overloading");
- return;
- }
- }
- else
- {
- m_gainIneffectiveSteps = 0;
- }
-
- double current = 0.0;
- if (!m_captureGain->gain(&current))
- return;
-
- // The bound that matters, and the one an earlier version of this got
- // wrong: how far down we have come in decibels, not where the slider sits.
- double currentDb = 0.0;
- if (m_haveGainDb && m_captureGain->gainDb(&currentDb))
- {
- if (m_gainStartDb - currentDb >= kMaxGainReductionDb)
- {
- closeGainCalibration(QString("already %1 dB down and still clipping")
- .arg(m_gainStartDb - currentDb, 0, 'f', 1));
- return;
- }
- }
-
- // Size the step to how badly it is clipping. A microphone left at maximum
- // is 20-30dB too hot and wants covering quickly; one that only tips over
- // on the loudest syllable wants nudging, not slamming.
- double factor;
- if (pinnedPercent >= 5.0)
- factor = 0.70;
- else if (pinnedPercent >= 0.5)
- factor = 0.85;
- else
- factor = 0.95;
-
- double next = current * factor;
- if (next >= current)
- {
- closeGainCalibration("control is already at its minimum");
- return;
- }
-
- if (!m_captureGain->setGain(next))
- {
- closeGainCalibration("the control would not move");
- return;
- }
-
- m_captureGainSteps++;
- m_lastPinnedPercent = pinnedPercent;
- m_gainSettleReports = kGainSettleReports;
-
- double achieved = 0.0;
- double newDb = 0.0;
- if (m_haveGainDb && m_captureGain->gainDb(&newDb))
- {
- achieved = currentDb - newDb;
- qDebug("Capture gain %.0f%% -> %.0f%% (%.1f dB, %.1f dB below the user's setting)",
- current * 100.0, next * 100.0, -achieved, m_gainStartDb - newDb);
-
- // A step that moved the slider but not the level means we are pushing
- // against the end of a control that no longer does anything.
- if (achieved < kMinEffectiveStepDb)
- m_gainIneffectiveSteps++;
-
- // Close on the step that crosses the ceiling rather than waiting for
- // another report, which would let one more step through first.
- if (m_gainStartDb - newDb >= kMaxGainReductionDb)
- closeGainCalibration(QString("reached the %1 dB limit")
- .arg(kMaxGainReductionDb, 0, 'f', 0));
- }
- else
- {
- qDebug("Capture gain %.0f%% -> %.0f%% (step %d of %d)",
- current * 100.0, next * 100.0, m_captureGainSteps, kMaxCaptureGainSteps);
- }
-
- // The echo path just changed by however many dB that was, so everything
- // the filter has learned about it is now wrong. Telling it to start over
- // costs a second of convergence; not telling it costs the rest of the call.
- if (m_echoCanceller)
- m_echoCanceller->reset();
-}
-
-void AudioVideoCallController::closeGainCalibration(const QString &why)
-{
- if (!m_gainCalibrating)
- return;
-
- m_gainCalibrating = false;
- qDebug() << "Microphone gain calibration finished:" << why;
-}
-
-void AudioVideoCallController::startCaptureGainControl(const QString &captureDevice)
-{
- m_captureGain = nullptr;
- m_savedCaptureGain = 0.0;
- m_haveSavedCaptureGain = false;
- m_captureGainSteps = 0;
- m_gainCalibrating = false;
- m_gainStartDb = 0.0;
- m_haveGainDb = false;
- m_gainIneffectiveSteps = 0;
- m_lastPinnedPercent = 0.0;
- m_gainSettleReports = 0;
-
- Settings st(config_dbname);
- if (!st.loadValue("audio.auto_capture_gain", 0).toBool())
- return;
-
- // Whichever driver the capture side is about to open the device with -
- // the mixer control has to be the one belonging to that device.
- const QString driver = AudioDevice::PlatformDriverName(CAPTURE);
-
- m_captureGain = CaptureGainControl::create(driver, captureDevice);
- if (!m_captureGain)
- return;
-
- if (!m_captureGain->gain(&m_savedCaptureGain))
- {
- // Without a reading there is nothing to put back afterwards, and
- // moving a control we cannot restore is not a trade worth making.
- qWarning() << "Could not read the capture gain - not taking it over";
- delete m_captureGain;
- m_captureGain = nullptr;
- return;
- }
-
- m_haveSavedCaptureGain = true;
- m_haveGainDb = m_captureGain->gainDb(&m_gainStartDb);
- m_gainCalibrating = true;
-
- if (m_haveGainDb)
- qDebug("Microphone gain calibration open on %s - at %.0f%% (%.1f dB), will cut at most %.0f dB",
- qPrintable(m_captureGain->description()), m_savedCaptureGain * 100.0,
- m_gainStartDb, kMaxGainReductionDb);
- else
- qDebug("Microphone gain calibration open on %s - at %.0f%%, no dB scale so bounded "
- "to %d steps",
- qPrintable(m_captureGain->description()), m_savedCaptureGain * 100.0,
- kMaxCaptureGainSteps);
}
-void AudioVideoCallController::stopCaptureGainControl()
-{
- if (!m_captureGain)
- return;
-
- // Put the user's own setting back. Only bother if we actually moved it,
- // so a call that never clipped cannot disturb a gain the user changed
- // themselves while it was running.
- if (m_haveSavedCaptureGain && m_captureGainSteps > 0)
- {
- if (m_captureGain->setGain(m_savedCaptureGain))
- qDebug() << "Capture gain restored to" << qRound(m_savedCaptureGain * 100.0) << "%";
- else
- qWarning() << "Could not restore the capture gain to"
- << qRound(m_savedCaptureGain * 100.0) << "% - it has been left turned down";
- }
-
- delete m_captureGain;
- m_captureGain = nullptr;
- m_haveSavedCaptureGain = false;
- m_captureGainSteps = 0;
- m_gainCalibrating = false;
-}
void AudioVideoCallController::incoming_audio_params(QByteArray peer_id, QString codec, quint16 sample_rate)
{
diff --git a/pica-client/audiovideocallcontroller.h b/pica-client/audiovideocallcontroller.h
index f7c9e37..0743601 100644
--- a/pica-client/audiovideocallcontroller.h
+++ b/pica-client/audiovideocallcontroller.h
@@ -22,7 +22,6 @@
#include <QElapsedTimer>
#include "callwindow.h"
#include "audiodevice.h"
-#include "capturegain.h"
#include "videodevice.h"
#include "toneplayer/toneplayer.h"
@@ -75,75 +74,10 @@ private:
QThread remotevideo_thread;
// Acoustic echo cancellation for the current call, shared by microphone
- // and output. Null when it is switched off in the settings, when
- // speexdsp could not start one, or between calls.
+ // and output. Null unless the "audio.echo_cancellation" setting is
+ // EchoCancellationOwn, and between calls.
EchoCancellerPtr m_echoCanceller;
- // The capture device's mixer control, held for the duration of a call so
- // a clipping microphone can be turned down. Null when the setting is off,
- // when the platform has no reachable control, or between calls.
- //
- // m_savedCaptureGain is where the control was found at the start of the
- // call, put back in stopAudioPipeline(): this is the user's own setting
- // and we are only borrowing it.
- CaptureGainControl *m_captureGain;
- double m_savedCaptureGain;
- bool m_haveSavedCaptureGain;
-
- // This is a calibration that runs at the start of a call and then stops -
- // NOT a gain control that rides the level for the call's duration. The
- // difference matters: an adaptive echo canceller can only track a
- // stationary echo path, and moving the microphone gain moves that path.
- // See the AGC comment in EchoCanceller's constructor, which says the same
- // thing about the reason speexdsp's own AGC is left switched off.
- //
- // So every adjustment resets the canceller, and once m_gainCalibrating is
- // false nothing touches the control again until the call ends.
- bool m_gainCalibrating;
- int m_captureGainSteps;
-
- // dB reading when calibration opened, and whether the platform would give
- // us one. Bounding the total cut in decibels rather than as a fraction of
- // the control's travel is the whole point: a fraction means different
- // things on a hardware mixer (linear in dB) and on a sound server or
- // WASAPI endpoint (roughly cubic), which is how an earlier version of this
- // managed to wind a microphone down to silence.
- double m_gainStartDb;
- bool m_haveGainDb;
-
- // Consecutive adjustments that moved the control but did not reduce the
- // clipping. Means the stage actually overloading is one we cannot reach -
- // an analog mic boost ahead of the converter, or Windows' separate
- // "Microphone Boost" - and that turning our control down further will
- // silence the user without ever fixing it.
- int m_gainIneffectiveSteps;
- double m_lastPinnedPercent;
-
- // Reports to ignore after an adjustment, while audio captured at the old
- // setting drains out of the pipeline.
- int m_gainSettleReports;
-
- static const int kMaxCaptureGainSteps = 6;
-
- // Bound used when the control will not report decibels at all, where we
- // are working blind and a wrong guess is what silences a microphone.
- static const int kMaxBlindGainSteps = 2;
- static const int kMaxGainIneffectiveSteps = 2;
- static const int kGainSettleReports = 1;
-
- // How long the calibration window stays open, in milliseconds of call
- // time. Long enough that someone who says nothing for the first few
- // seconds still gets calibrated, short enough to be over before the
- // conversation proper.
- static const int kGainCalibrationMs = 20000;
-
- // Hard ceiling on the total cut, in dB below where the user had it. Past
- // this the problem is not one we are going to fix by turning a knob.
- static constexpr double kMaxGainReductionDb = 24.0;
-
- // An adjustment that achieves less than this much is not doing anything.
- static constexpr double kMinEffectiveStepDb = 0.5;
-
// Sequence number / capture-clock timestamp stamped onto outgoing
// 0x76 audio packets (see PICA_PROTO_CALL_PACKET_HDRSIZE).
quint16 m_audioSeq;
@@ -161,15 +95,6 @@ private:
// Reassembly of incoming 0x77 fragments into whole encoded frames.
VideoFrameAssembler m_videoAssembler;
- // Take over the capture device's mixer gain for the duration of a call,
- // and hand it back. Both are no-ops when "audio.auto_capture_gain" is off
- // or the platform has no control we can reach.
- void startCaptureGainControl(const QString &captureDevice);
- void stopCaptureGainControl();
-
- // Ends the calibration window, logging why. Idempotent.
- void closeGainCalibration(const QString &why);
-
void playEarpieceTone(void (TonePlayer::*tone)());
void playRingTone();
void stopTones();
diff --git a/pica-client/capturegain.cpp b/pica-client/capturegain.cpp
deleted file mode 100644
index 8f89718..0000000
--- a/pica-client/capturegain.cpp
+++ /dev/null
@@ -1,533 +0,0 @@
-/*
- (c) Copyright 2012 - 2026 Anton Sviridenko
- https://picapica.im
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-#include "capturegain.h"
-
-#include <QDebug>
-#include <QtGlobal>
-#include <QStringList>
-#include <cmath>
-
-#ifdef Q_OS_LINUX
-#include <alsa/asoundlib.h>
-#endif
-
-#ifdef HAVE_LIBPULSE
-#include <pulse/pulseaudio.h>
-#endif
-
-// Per platform factories, defined next to the implementations themselves -
-// same arrangement as NativeAudioBackend::create().
-#ifdef Q_OS_WIN
-CaptureGainControl *pica_create_wasapi_capture_gain(const QString &device);
-#endif
-
-#if defined(Q_OS_MACOS) || defined(Q_OS_MAC)
-CaptureGainControl *pica_create_coreaudio_capture_gain(const QString &device);
-#endif
-
-CaptureGainControl::~CaptureGainControl()
-{
-}
-
-static double clampGain(double v)
-{
- if (v < 0.0)
- return 0.0;
- if (v > 1.0)
- return 1.0;
- return v;
-}
-
-// PulseAudio is the one backend whose control goes past its nominal maximum -
-// PA_VOLUME_NORM is 100% and the server will happily sit above it. Clamping a
-// reading to 1.0 there would mean handing a user who had their source at 130%
-// back at 100% when the call ended, so that path keeps whatever it read and
-// only guards the bottom of the range and a runaway top.
-static double clampGainAmplified(double v)
-{
- if (v < 0.0)
- return 0.0;
- if (v > 2.5)
- return 2.5;
- return v;
-}
-
-#ifdef Q_OS_LINUX
-
-// The mixer lives on a card, but what we are handed is a PCM name. The two
-// namespaces overlap only by convention, so this covers the forms that
-// actually turn up in the device list and falls back to the default card for
-// anything else - which is right for plain "default" and no worse than
-// nothing for the exotic cases.
-static QString alsaCardFromPcm(const QString &pcm)
-{
- int card = pcm.indexOf(QLatin1String("CARD="));
- if (card >= 0)
- {
- QString name = pcm.mid(card + 5);
- int comma = name.indexOf(QLatin1Char(','));
- if (comma >= 0)
- name = name.left(comma);
- if (!name.isEmpty())
- return QStringLiteral("hw:") + name;
- }
-
- // "hw:1,0", "plughw:CARD=PCH,DEV=0", "dsnoop:1" - everything before the
- // first comma after the colon names the card.
- int colon = pcm.indexOf(QLatin1Char(':'));
- if (colon >= 0)
- {
- QString rest = pcm.mid(colon + 1);
- int comma = rest.indexOf(QLatin1Char(','));
- if (comma >= 0)
- rest = rest.left(comma);
- if (!rest.isEmpty())
- return QStringLiteral("hw:") + rest;
- }
-
- return QStringLiteral("default");
-}
-
-class AlsaCaptureGain : public CaptureGainControl
-{
-public:
- AlsaCaptureGain() : m_mixer(nullptr), m_elem(nullptr), m_min(0), m_max(0) {}
-
- ~AlsaCaptureGain() override
- {
- if (m_mixer)
- snd_mixer_close(m_mixer);
- }
-
- bool open(const QString &pcmName)
- {
- // The card the PCM name points at, and then the real cards as
- // fallbacks. "default" is not a card at all on a machine running a
- // sound server: attaching to it lands on the server's own software
- // volume, which is applied to samples the converter has already
- // digitised. That control cannot undo clipping - scaling a clipped
- // buffer down just gives a quieter clipped buffer - so calibrating
- // against it would turn the microphone down forever without the
- // clipping ever going away. Only a gain ahead of the ADC is any use
- // here, and tryCard() insists on one.
- QStringList candidates;
- candidates << alsaCardFromPcm(pcmName);
-
- if (candidates.first() == QLatin1String("default"))
- {
- for (int card = 0; card < 8; card++)
- candidates << (QStringLiteral("hw:") + QString::number(card));
- }
-
- for (int i = 0; i < candidates.size(); i++)
- {
- if (tryCard(candidates[i]))
- return true;
- }
-
- return false;
- }
-
- bool gain(double *out) override
- {
- if (!m_elem || !out)
- return false;
-
- // The kernel may have moved the control under us - another mixer app,
- // or the user's own volume keys.
- snd_mixer_handle_events(m_mixer);
-
- long v = 0;
- if (snd_mixer_selem_get_capture_volume(m_elem, SND_MIXER_SCHN_FRONT_LEFT, &v) < 0 &&
- snd_mixer_selem_get_capture_volume(m_elem, SND_MIXER_SCHN_MONO, &v) < 0)
- return false;
-
- *out = clampGain(double(v - m_min) / double(m_max - m_min));
- return true;
- }
-
- bool gainDb(double *out) override
- {
- if (!m_elem || !out)
- return false;
-
- snd_mixer_handle_events(m_mixer);
-
- // ALSA reports hundredths of a decibel. Not every element has a dB
- // mapping - the software controls a sound server exposes often do not.
- long v = 0;
- if (snd_mixer_selem_get_capture_dB(m_elem, SND_MIXER_SCHN_FRONT_LEFT, &v) < 0 &&
- snd_mixer_selem_get_capture_dB(m_elem, SND_MIXER_SCHN_MONO, &v) < 0)
- return false;
-
- *out = double(v) / 100.0;
- return true;
- }
-
- bool setGain(double value) override
- {
- if (!m_elem)
- return false;
-
- long v = m_min + (long)llround(clampGain(value) * double(m_max - m_min));
- return snd_mixer_selem_set_capture_volume_all(m_elem, v) >= 0;
- }
-
- QString description() const override { return m_name; }
-
-private:
- // A candidate has to have a decibel range as well as a volume range. That
- // is what separates a converter gain from a sound server's software fader:
- // the hardware control knows what its steps are worth in dB, the software
- // one only has a slider position. It is also what lets the caller bound
- // the total cut in dB, which is the only unit that means the same thing on
- // every backend.
- bool tryCard(const QString &card)
- {
- if (m_mixer)
- {
- snd_mixer_close(m_mixer);
- m_mixer = nullptr;
- m_elem = nullptr;
- }
-
- if (snd_mixer_open(&m_mixer, 0) < 0)
- {
- m_mixer = nullptr;
- return false;
- }
-
- const QByteArray cardUtf8 = card.toUtf8();
- if (snd_mixer_attach(m_mixer, cardUtf8.constData()) < 0 ||
- snd_mixer_selem_register(m_mixer, nullptr, nullptr) < 0 ||
- snd_mixer_load(m_mixer) < 0)
- return false;
-
- // "Capture" is what the recording gain is called on essentially every
- // card that has one; "Mic" turns up on the ones that do not separate
- // the input selector from the gain.
- m_elem = findNamed("Capture");
- if (!m_elem)
- m_elem = findNamed("Mic");
- if (!m_elem)
- m_elem = findNamed("Digital");
-
- if (!m_elem)
- return false;
-
- if (snd_mixer_selem_get_capture_volume_range(m_elem, &m_min, &m_max) < 0 || m_max <= m_min)
- return false;
-
- long dbMin = 0, dbMax = 0;
- if (snd_mixer_selem_get_capture_dB_range(m_elem, &dbMin, &dbMax) < 0 || dbMax <= dbMin)
- {
- m_elem = nullptr;
- return false;
- }
-
- m_name = QString::fromUtf8(snd_mixer_selem_get_name(m_elem)) +
- QStringLiteral(" on ") + card;
- return true;
- }
-
- snd_mixer_elem_t *findNamed(const char *want)
- {
- for (snd_mixer_elem_t *e = snd_mixer_first_elem(m_mixer); e; e = snd_mixer_elem_next(e))
- {
- if (!snd_mixer_selem_is_active(e) || !snd_mixer_selem_has_capture_volume(e))
- continue;
- if (qstrcmp(snd_mixer_selem_get_name(e), want) == 0)
- return e;
- }
- return nullptr;
- }
-
- snd_mixer_t *m_mixer;
- snd_mixer_elem_t *m_elem;
- long m_min;
- long m_max;
- QString m_name;
-};
-
-#endif // Q_OS_LINUX
-
-#ifdef HAVE_LIBPULSE
-
-// PulseAudio's introspection and its setters are both asynchronous, so this
-// keeps a connected context with a private mainloop and pumps it until each
-// operation has called back. Same shape as enumeratePulse() in
-// audiodevice.cpp, but long lived, since a call makes several of these.
-class PulseCaptureGain : public CaptureGainControl
-{
-public:
- PulseCaptureGain()
- : m_mainloop(nullptr), m_ctx(nullptr), m_channels(1),
- m_pending(false), m_ok(false), m_value(0.0), m_valueDb(0.0) {}
-
- ~PulseCaptureGain() override
- {
- if (m_ctx)
- {
- pa_context_disconnect(m_ctx);
- pa_context_unref(m_ctx);
- }
- if (m_mainloop)
- pa_mainloop_free(m_mainloop);
- }
-
- bool open(const QString &device)
- {
- m_mainloop = pa_mainloop_new();
- if (!m_mainloop)
- return false;
-
- m_ctx = pa_context_new(pa_mainloop_get_api(m_mainloop), "pica-client");
- if (!m_ctx)
- return false;
-
- if (pa_context_connect(m_ctx, nullptr, PA_CONTEXT_NOFLAGS, nullptr) < 0)
- return false;
-
- // Five seconds at 50ms a turn, so a server that never answers cannot
- // wedge the call setup.
- for (int i = 0; i < 100; i++)
- {
- pa_context_state_t st = pa_context_get_state(m_ctx);
- if (st == PA_CONTEXT_READY)
- break;
- if (st == PA_CONTEXT_FAILED || st == PA_CONTEXT_TERMINATED)
- return false;
- if (!pump())
- return false;
- }
-
- if (pa_context_get_state(m_ctx) != PA_CONTEXT_READY)
- return false;
-
- m_source = device;
- if (m_source.isEmpty() || m_source == QLatin1String("default"))
- {
- if (!resolveDefaultSource())
- return false;
- }
-
- // Also establishes the channel count, which setGain() needs.
- double ignored;
- return gain(&ignored);
- }
-
- bool gain(double *out) override
- {
- if (!m_ctx || m_source.isEmpty())
- return false;
-
- const QByteArray name = m_source.toUtf8();
- m_pending = true;
- m_ok = false;
-
- pa_operation *op = pa_context_get_source_info_by_name(m_ctx, name.constData(),
- sourceInfoCb, this);
- if (!await(op))
- return false;
-
- if (m_ok && out)
- *out = m_value;
- return m_ok;
- }
-
- // pa_sw_volume_to_dB() returns -inf for a muted control, which is true but
- // useless as a number to do arithmetic on.
- bool gainDb(double *out) override
- {
- if (!out)
- return false;
-
- double ignored;
- if (!gain(&ignored))
- return false;
-
- if (m_valueDb < -200.0 || m_valueDb > 200.0)
- return false;
-
- *out = m_valueDb;
- return true;
- }
-
- bool setGain(double value) override
- {
- if (!m_ctx || m_source.isEmpty())
- return false;
-
- pa_cvolume cv;
- pa_cvolume_set(&cv, m_channels ? m_channels : 1,
- (pa_volume_t)llround(clampGainAmplified(value) * double(PA_VOLUME_NORM)));
-
- const QByteArray name = m_source.toUtf8();
- m_pending = true;
- m_ok = false;
-
- pa_operation *op = pa_context_set_source_volume_by_name(m_ctx, name.constData(), &cv,
- successCb, this);
- if (!await(op))
- return false;
-
- return m_ok;
- }
-
- QString description() const override
- {
- return m_source + QStringLiteral(" (PulseAudio)");
- }
-
-private:
- bool pump()
- {
- if (pa_mainloop_prepare(m_mainloop, 50 * 1000) < 0)
- return false;
- if (pa_mainloop_poll(m_mainloop) < 0)
- return false;
- if (pa_mainloop_dispatch(m_mainloop) < 0)
- return false;
- return true;
- }
-
- // Runs the loop until the callback for op has fired, then reaps it.
- bool await(pa_operation *op)
- {
- if (!op)
- {
- m_pending = false;
- return false;
- }
-
- for (int i = 0; i < 100 && m_pending; i++)
- {
- if (!pump())
- break;
- }
-
- bool finished = !m_pending;
- m_pending = false;
- pa_operation_unref(op);
- return finished;
- }
-
- bool resolveDefaultSource()
- {
- m_pending = true;
- m_ok = false;
-
- pa_operation *op = pa_context_get_server_info(m_ctx, serverInfoCb, this);
- if (!await(op))
- return false;
-
- return m_ok && !m_source.isEmpty();
- }
-
- static void serverInfoCb(pa_context *, const pa_server_info *info, void *userdata)
- {
- PulseCaptureGain *self = (PulseCaptureGain *)userdata;
-
- if (info && info->default_source_name)
- {
- self->m_source = QString::fromUtf8(info->default_source_name);
- self->m_ok = true;
- }
- self->m_pending = false;
- }
-
- static void sourceInfoCb(pa_context *, const pa_source_info *info, int eol, void *userdata)
- {
- PulseCaptureGain *self = (PulseCaptureGain *)userdata;
-
- if (eol)
- {
- self->m_pending = false;
- return;
- }
-
- if (!info)
- return;
-
- self->m_channels = info->volume.channels;
- // The loudest channel, so a step down is a step down for all of them
- // rather than only for whichever happens to be quietest.
- self->m_value = clampGainAmplified(double(pa_cvolume_max(&info->volume)) / double(PA_VOLUME_NORM));
- self->m_valueDb = pa_sw_volume_to_dB(pa_cvolume_max(&info->volume));
- self->m_ok = true;
- }
-
- static void successCb(pa_context *, int success, void *userdata)
- {
- PulseCaptureGain *self = (PulseCaptureGain *)userdata;
- self->m_ok = (success != 0);
- self->m_pending = false;
- }
-
- pa_mainloop *m_mainloop;
- pa_context *m_ctx;
- QString m_source;
- uint8_t m_channels;
-
- bool m_pending;
- bool m_ok;
- double m_value;
- double m_valueDb;
-};
-
-#endif // HAVE_LIBPULSE
-
-CaptureGainControl *CaptureGainControl::create(const QString &driver, const QString &device)
-{
-#ifdef Q_OS_WIN
- if (driver == QLatin1String("platform-wasapi"))
- return pica_create_wasapi_capture_gain(device);
-#endif
-
-#if defined(Q_OS_MACOS) || defined(Q_OS_MAC)
- if (driver == QLatin1String("platform-coreaudio"))
- return pica_create_coreaudio_capture_gain(device);
-#endif
-
-#ifdef HAVE_LIBPULSE
- if (driver == QLatin1String("pulse"))
- {
- PulseCaptureGain *g = new PulseCaptureGain();
- if (g->open(device))
- return g;
- delete g;
- qWarning() << "No PulseAudio volume control for capture source" << device;
- return nullptr;
- }
-#endif
-
-#ifdef Q_OS_LINUX
- if (driver == QLatin1String("alsa"))
- {
- AlsaCaptureGain *g = new AlsaCaptureGain();
- if (g->open(device))
- return g;
- delete g;
- qWarning() << "No ALSA capture volume control for device" << device;
- return nullptr;
- }
-#endif
-
- Q_UNUSED(driver)
- Q_UNUSED(device)
- return nullptr;
-}
diff --git a/pica-client/capturegain.h b/pica-client/capturegain.h
deleted file mode 100644
index a0d3bdc..0000000
--- a/pica-client/capturegain.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- (c) Copyright 2012 - 2026 Anton Sviridenko
- https://picapica.im
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-#ifndef CAPTUREGAIN_H
-#define CAPTUREGAIN_H
-
-#include <QString>
-#include <QtGlobal>
-
-// The capture device's own gain control - the mixer slider, not a multiplier
-// applied to samples we already have.
-//
-// The distinction is the entire point of this class. Clipping happens in the
-// converter, before any buffer reaches us; by then the peaks are flat and the
-// harmonics they generated are in the signal for good. Scaling that buffer
-// down produces a quieter clipped buffer and nothing else - in particular it
-// does not help the echo canceller, whose adaptive filter is linear and
-// therefore scale invariant. What defeats the filter is the non-linearity, and
-// the only place to prevent one is ahead of the ADC. That is this control.
-//
-// Gain is expressed as a 0.0 to 1.0 fraction of the control's travel, which is
-// what WASAPI and CoreAudio use natively and what ALSA and PulseAudio's ranges
-// map onto cleanly. The mapping from that fraction to decibels is the
-// platform's business and is not the same on all of them, so callers should
-// treat a step as "somewhat quieter" and close the loop by measuring again
-// rather than doing arithmetic in dB.
-//
-// Not thread safe, and deliberately so: AudioVideoCallController owns one of
-// these and touches it only from the GUI thread, driven by a queued signal
-// from the capture thread. Some of the implementations below need a message
-// loop or a COM apartment and have no business being called from an audio
-// callback.
-class CaptureGainControl
-{
-public:
- virtual ~CaptureGainControl();
-
- // Current position of the control, 0.0 to 1.0. False if it could not be
- // read, in which case *out is untouched.
- virtual bool gain(double *out) = 0;
-
- // The same position in decibels, where the platform can say. This is what
- // a caller should bound itself by: the fraction above is a position on a
- // slider whose relationship to loudness differs per backend - linear in dB
- // on an ALSA hardware control, roughly cubic on PulseAudio and WASAPI - so
- // "5% of travel" is a mild cut on one and near silence on another. Decibels
- // mean the same thing everywhere.
- //
- // False when the platform will not say, and then the caller has nothing to
- // go on but a step count.
- virtual bool gainDb(double *out) { Q_UNUSED(out) return false; }
-
- // Moves the control. Values outside 0.0 to 1.0 are clamped.
- virtual bool setGain(double value) = 0;
-
- // What was actually found - the mixer element or endpoint name - for the
- // log line that says the gain is being taken over.
- virtual QString description() const = 0;
-
- // driver is a name from AudioDevice::PlatformDriverName(), device the
- // capture device string that goes with it. Returns null when this build
- // or this platform has no way to reach the control, which is not an
- // error: the caller carries on without automatic gain and the clipping
- // warning still gets shown.
- static CaptureGainControl *create(const QString &driver, const QString &device);
-};
-
-#endif // CAPTUREGAIN_H
diff --git a/pica-client/capturegain_coreaudio.cpp b/pica-client/capturegain_coreaudio.cpp
deleted file mode 100644
index 919d08d..0000000
--- a/pica-client/capturegain_coreaudio.cpp
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- (c) Copyright 2012 - 2026 Anton Sviridenko
- https://picapica.im
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-#include "capturegain.h"
-
-#if defined(Q_OS_MACOS) || defined(Q_OS_MAC)
-
-#include <QDebug>
-#include <QVector>
-
-#include <CoreAudio/CoreAudio.h>
-#include <CoreFoundation/CoreFoundation.h>
-
-// Small local copies of the device lookup helpers in
-// nativeaudio_coreaudio.cpp. They are static over there and this is the only
-// other place that needs them; duplicating three short functions beats
-// exporting an internal API out of that file.
-static QVector<AudioDeviceID> allInputCapableDevices()
-{
- QVector<AudioDeviceID> devices;
-
- AudioObjectPropertyAddress addr;
- addr.mSelector = kAudioHardwarePropertyDevices;
- addr.mScope = kAudioObjectPropertyScopeGlobal;
- addr.mElement = kAudioObjectPropertyElementMaster;
-
- UInt32 size = 0;
- if (AudioObjectGetPropertyDataSize(kAudioObjectSystemObject, &addr, 0, nullptr, &size) != noErr)
- return devices;
-
- devices.resize(size / sizeof(AudioDeviceID));
- if (devices.isEmpty())
- return devices;
-
- if (AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, 0, nullptr, &size,
- devices.data()) != noErr)
- devices.clear();
-
- return devices;
-}
-
-static QString deviceUidOf(AudioDeviceID dev)
-{
- AudioObjectPropertyAddress addr;
- addr.mSelector = kAudioDevicePropertyDeviceUID;
- addr.mScope = kAudioObjectPropertyScopeGlobal;
- addr.mElement = kAudioObjectPropertyElementMaster;
-
- CFStringRef uid = nullptr;
- UInt32 size = sizeof(uid);
- if (AudioObjectGetPropertyData(dev, &addr, 0, nullptr, &size, &uid) != noErr || !uid)
- return QString();
-
- char buf[512] = {0};
- QString result;
- if (CFStringGetCString(uid, buf, sizeof(buf), kCFStringEncodingUTF8))
- result = QString::fromUtf8(buf);
-
- CFRelease(uid);
- return result;
-}
-
-static AudioDeviceID defaultInputDevice()
-{
- AudioObjectPropertyAddress addr;
- addr.mSelector = kAudioHardwarePropertyDefaultInputDevice;
- addr.mScope = kAudioObjectPropertyScopeGlobal;
- addr.mElement = kAudioObjectPropertyElementMaster;
-
- AudioDeviceID dev = kAudioObjectUnknown;
- UInt32 size = sizeof(dev);
- if (AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, 0, nullptr, &size, &dev) != noErr)
- return kAudioObjectUnknown;
-
- return dev;
-}
-
-// The input volume of a CoreAudio device.
-//
-// Which element carries it varies: some devices expose a master (element 0),
-// some only per channel controls, and plenty of built in microphones expose
-// none at all and are simply not adjustable. open() works that out once and
-// remembers the answer; a device with nothing settable fails to open, and the
-// caller falls back to warning the user.
-class CoreAudioCaptureGain : public CaptureGainControl
-{
-public:
- CoreAudioCaptureGain() : m_dev(kAudioObjectUnknown), m_master(false) {}
-
- bool open(const QString &device)
- {
- if (device.isEmpty() || device == QLatin1String("default"))
- {
- m_dev = defaultInputDevice();
- }
- else
- {
- QVector<AudioDeviceID> devices = allInputCapableDevices();
- for (int i = 0; i < devices.size(); i++)
- {
- if (deviceUidOf(devices[i]) == device)
- {
- m_dev = devices[i];
- break;
- }
- }
-
- // Fall back rather than give up - the same thing the audio unit
- // does when a remembered device has been unplugged.
- if (m_dev == kAudioObjectUnknown)
- m_dev = defaultInputDevice();
- }
-
- if (m_dev == kAudioObjectUnknown)
- return false;
-
- if (settable(0))
- {
- m_master = true;
- m_channels.clear();
- }
- else
- {
- // Stereo inputs are the common case; anything past the second
- // channel is rare enough on a microphone not to chase.
- for (UInt32 ch = 1; ch <= 2; ch++)
- {
- if (settable(ch))
- m_channels << ch;
- }
-
- if (m_channels.isEmpty())
- {
- qWarning() << "CoreAudio: input device has no adjustable volume";
- return false;
- }
- }
-
- m_device = device.isEmpty() ? QStringLiteral("default") : device;
- return true;
- }
-
- bool gain(double *out) override
- {
- if (!out || m_dev == kAudioObjectUnknown)
- return false;
-
- // The loudest of the channels, so stepping down moves all of them.
- Float32 best = -1.0f;
-
- if (m_master)
- {
- if (!readElement(0, &best))
- return false;
- }
- else
- {
- for (int i = 0; i < m_channels.size(); i++)
- {
- Float32 v = 0.0f;
- if (readElement(m_channels[i], &v) && v > best)
- best = v;
- }
- }
-
- if (best < 0.0f)
- return false;
-
- *out = best;
- return true;
- }
-
- // kAudioDevicePropertyVolumeDecibels is the same control as the scalar
- // above, reported on a decibel scale. Devices that expose the scalar do not
- // always expose this one, hence the plain false rather than a fallback.
- bool gainDb(double *out) override
- {
- if (!out || m_dev == kAudioObjectUnknown)
- return false;
-
- AudioObjectPropertyAddress addr;
- addr.mSelector = kAudioDevicePropertyVolumeDecibels;
- addr.mScope = kAudioObjectPropertyScopeInput;
- addr.mElement = m_master ? 0 : (m_channels.isEmpty() ? 0 : m_channels[0]);
-
- if (!AudioObjectHasProperty(m_dev, &addr))
- return false;
-
- Float32 db = 0.0f;
- UInt32 size = sizeof(db);
- if (AudioObjectGetPropertyData(m_dev, &addr, 0, nullptr, &size, &db) != noErr)
- return false;
-
- *out = db;
- return true;
- }
-
- bool setGain(double value) override
- {
- if (m_dev == kAudioObjectUnknown)
- return false;
-
- Float32 v = (Float32)(value < 0.0 ? 0.0 : (value > 1.0 ? 1.0 : value));
-
- if (m_master)
- return writeElement(0, v);
-
- bool any = false;
- for (int i = 0; i < m_channels.size(); i++)
- {
- if (writeElement(m_channels[i], v))
- any = true;
- }
-
- return any;
- }
-
- QString description() const override
- {
- return m_device + QStringLiteral(" (CoreAudio input volume)");
- }
-
-private:
- static AudioObjectPropertyAddress addressFor(UInt32 element)
- {
- AudioObjectPropertyAddress addr;
- addr.mSelector = kAudioDevicePropertyVolumeScalar;
- addr.mScope = kAudioObjectPropertyScopeInput;
- addr.mElement = element;
- return addr;
- }
-
- bool settable(UInt32 element)
- {
- AudioObjectPropertyAddress addr = addressFor(element);
-
- if (!AudioObjectHasProperty(m_dev, &addr))
- return false;
-
- Boolean writable = false;
- if (AudioObjectIsPropertySettable(m_dev, &addr, &writable) != noErr)
- return false;
-
- return writable;
- }
-
- bool readElement(UInt32 element, Float32 *out)
- {
- AudioObjectPropertyAddress addr = addressFor(element);
- UInt32 size = sizeof(*out);
- return AudioObjectGetPropertyData(m_dev, &addr, 0, nullptr, &size, out) == noErr;
- }
-
- bool writeElement(UInt32 element, Float32 value)
- {
- AudioObjectPropertyAddress addr = addressFor(element);
- return AudioObjectSetPropertyData(m_dev, &addr, 0, nullptr, sizeof(value), &value) == noErr;
- }
-
- AudioDeviceID m_dev;
- bool m_master;
- QVector<UInt32> m_channels;
- QString m_device;
-};
-
-CaptureGainControl *pica_create_coreaudio_capture_gain(const QString &device)
-{
- CoreAudioCaptureGain *g = new CoreAudioCaptureGain();
-
- if (g->open(device))
- return g;
-
- delete g;
- return nullptr;
-}
-
-#endif // Q_OS_MACOS
diff --git a/pica-client/capturegain_wasapi.cpp b/pica-client/capturegain_wasapi.cpp
deleted file mode 100644
index 49ea84d..0000000
--- a/pica-client/capturegain_wasapi.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- (c) Copyright 2012 - 2026 Anton Sviridenko
- https://picapica.im
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, version 3.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <https://www.gnu.org/licenses/>.
-*/
-#include "capturegain.h"
-
-#ifdef Q_OS_WIN
-
-#include <QDebug>
-
-// Deliberately NOT defining INITGUID here, unlike nativeaudio_wasapi.cpp: that
-// file already emits the definitions for the endpoint GUIDs, and a second copy
-// in this translation unit would collide at link time. The interface IDs below
-// come from __uuidof() instead, which mingw-w64 supports because its headers
-// declare these interfaces with __CRT_UUID_DECL.
-#include <windows.h>
-#include <objbase.h>
-#include <mmdeviceapi.h>
-#include <endpointvolume.h>
-
-static QString hrText(HRESULT hr)
-{
- return QString("0x%1").arg((quint32)hr, 8, 16, QLatin1Char('0'));
-}
-
-// The endpoint volume, which is the slider under Settings > Sound > Input.
-//
-// Note this is not the same control as "Microphone Boost" - that one is a
-// driver specific APO with no uniform API, and is usually the thing actually
-// responsible when a mic clips at every level. Turning the endpoint volume
-// down still reduces what reaches the converter on the great majority of
-// devices, so it is the right control to reach for; it just may not have as
-// much travel as the ALSA case.
-class WasapiCaptureGain : public CaptureGainControl
-{
-public:
- WasapiCaptureGain() : m_vol(nullptr), m_uninitCom(false) {}
-
- ~WasapiCaptureGain() override
- {
- if (m_vol)
- m_vol->Release();
- if (m_uninitCom)
- CoUninitialize();
- }
-
- bool open(const QString &device)
- {
- HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
- // S_FALSE means this thread was already in an apartment, and
- // RPC_E_CHANGED_MODE that it is in a different one - in both cases COM
- // is usable and we must not be the one to tear it down.
- if (SUCCEEDED(hr))
- m_uninitCom = true;
- else if (hr != RPC_E_CHANGED_MODE)
- return false;
-
- IMMDeviceEnumerator *devEnum = nullptr;
- hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
- __uuidof(IMMDeviceEnumerator), (void **)&devEnum);
- if (FAILED(hr) || !devEnum)
- return false;
-
- IMMDevice *dev = nullptr;
- if (device.isEmpty() || device == QLatin1String("default"))
- {
- // eCommunications to match what the capture stream itself opens in
- // WasapiBackend::resolveDevice() - otherwise on a machine where the
- // call endpoint and the console endpoint differ we would be turning
- // down a device nobody is recording from.
- hr = devEnum->GetDefaultAudioEndpoint(eCapture, eCommunications, &dev);
- }
- else
- {
- hr = devEnum->GetDevice((LPCWSTR)device.utf16(), &dev);
- }
-
- devEnum->Release();
-
- if (FAILED(hr) || !dev)
- return false;
-
- hr = dev->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, nullptr, (void **)&m_vol);
- dev->Release();
-
- if (FAILED(hr) || !m_vol)
- {
- qWarning() << "WASAPI: capture endpoint has no volume control:" << hrText(hr);
- m_vol = nullptr;
- return false;
- }
-
- m_device = device.isEmpty() ? QStringLiteral("default") : device;
- return true;
- }
-
- bool gain(double *out) override
- {
- if (!m_vol || !out)
- return false;
-
- float level = 0.0f;
- if (FAILED(m_vol->GetMasterVolumeLevelScalar(&level)))
- return false;
-
- *out = level;
- return true;
- }
-
- // WASAPI keeps a real decibel value alongside the scalar, so this needs no
- // conversion of our own.
- bool gainDb(double *out) override
- {
- if (!m_vol || !out)
- return false;
-
- float db = 0.0f;
- if (FAILED(m_vol->GetMasterVolumeLevel(&db)))
- return false;
-
- *out = db;
- return true;
- }
-
- bool setGain(double value) override
- {
- if (!m_vol)
- return false;
-
- float level = (float)(value < 0.0 ? 0.0 : (value > 1.0 ? 1.0 : value));
- return SUCCEEDED(m_vol->SetMasterVolumeLevelScalar(level, nullptr));
- }
-
- QString description() const override
- {
- return m_device + QStringLiteral(" (WASAPI endpoint volume)");
- }
-
-private:
- IAudioEndpointVolume *m_vol;
- bool m_uninitCom;
- QString m_device;
-};
-
-CaptureGainControl *pica_create_wasapi_capture_gain(const QString &device)
-{
- WasapiCaptureGain *g = new WasapiCaptureGain();
-
- if (g->open(device))
- return g;
-
- delete g;
- return nullptr;
-}
-
-#endif // Q_OS_WIN
diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp
index a951b40..8685936 100644
--- a/pica-client/dialogs/settingsdialog.cpp
+++ b/pica-client/dialogs/settingsdialog.cpp
@@ -195,18 +195,38 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
audioTestStatus = new QLabel(this);
audioTestStatus->setAlignment(Qt::AlignCenter);
- cbEchoCancel = new QCheckBox(tr("Cancel acoustic echo"), this);
- cbEchoCancel->setToolTip(tr("Removes the sound of the other person coming back out of your "
- "microphone, which is what makes them hear themselves when you "
- "are on speakers. Has no effect where the operating system "
- "already does it."));
-
- cbAutoCaptureGain = new QCheckBox(tr("Automatic microphone gain"), this);
- cbAutoCaptureGain->setToolTip(tr("Turns the microphone's own recording level down during a "
- "call if it is loud enough to distort, and puts it back "
- "afterwards. A microphone set too loud clips, and clipping "
- "cannot be repaired later - it also stops echo cancellation "
- "working."));
+ // Echo cancellation: removing the sound of the other person coming back
+ // out of the microphone, which is what makes them hear themselves when
+ // the call is on speakers. Exactly one of the three - two cancellers in
+ // series is worse than either alone, since the second one is left
+ // chasing a reference that is no longer in the signal.
+ QGroupBox *gbEchoCancel = new QGroupBox(tr("Audio echo cancellation"), this);
+ QVBoxLayout *echoCancelLayout = new QVBoxLayout();
+
+ // Parented to the group box, not the dialog: QRadioButton's automatic
+ // exclusivity groups by parent widget, and these three must not join the
+ // direct connection or multiple login buttons.
+ rbEchoCancelOwn = new QRadioButton(tr("Use own echo cancellation"), gbEchoCancel);
+ rbEchoCancelOwn->setToolTip(tr("Cancels the echo inside Pica Pica, using WebRTC's AEC3. "
+ "Works with any microphone and any sound system, and is the "
+ "only option where the platform provides nothing."));
+
+ rbEchoCancelPlatform = new QRadioButton(tr("Use platform provided echo cancellation"), gbEchoCancel);
+
+ rbEchoCancelNone = new QRadioButton(tr("Disable echo cancellation"), gbEchoCancel);
+ rbEchoCancelNone->setToolTip(tr("Nothing is removed from the microphone signal. Right with a "
+ "headset, where there is no echo to cancel, and worth trying "
+ "if cancellation is cutting into your speech."));
+
+ echoCancelLayout->addWidget(rbEchoCancelOwn);
+ echoCancelLayout->addWidget(rbEchoCancelPlatform);
+ echoCancelLayout->addWidget(rbEchoCancelNone);
+ gbEchoCancel->setLayout(echoCancelLayout);
+
+ // Fills in the platform option's tooltip and greys it out where there is
+ // no such thing. loadSettings() runs afterwards and is what actually
+ // picks one of the three.
+ updateEchoCancellationChoices();
btRingTest = new QPushButton(tr("Test ring 🔔"), this);
connect(btRingTest, SIGNAL(clicked()), this, SLOT(toggleRingTest()));
@@ -224,8 +244,7 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
audiodevlayout->addWidget(audioCaptureDev);
audiodevlayout->addWidget(lbAudioPlaybackDev);
audiodevlayout->addWidget(audioPlaybackDev);
- audiodevlayout->addWidget(cbEchoCancel);
- audiodevlayout->addWidget(cbAutoCaptureGain);
+ audiodevlayout->addWidget(gbEchoCancel);
audiodevlayout->addWidget(btAudioTest);
audiodevlayout->addWidget(audioTestStatus);
audiodevlayout->addWidget(lbAudioRingDev);
@@ -908,6 +927,43 @@ void SettingsDialog::audioDriverChanged(int index)
fillAudioCaptureDevices();
fillAudioPlaybackDevices();
fillAudioRingDevices();
+
+ // ALSA has no echo cancellation of its own and PulseAudio does, so the
+ // platform option comes and goes with this choice.
+ updateEchoCancellationChoices();
+}
+
+void SettingsDialog::updateEchoCancellationChoices()
+{
+ // PlatformDriverName() reads the driver SetLinuxDriverName() published,
+ // which audioDriverChanged() has already updated by the time it calls
+ // this - so the answer follows the combo box rather than the saved
+ // setting.
+ const QString driver = AudioDevice::PlatformDriverName(CAPTURE);
+ const bool available = AudioDevice::PlatformEchoCancellationAvailable(driver);
+
+ rbEchoCancelPlatform->setEnabled(available);
+
+ if (available)
+ {
+ rbEchoCancelPlatform->setToolTip(tr("Lets the operating system or sound server cancel the "
+ "echo instead. Where it exists this is the better "
+ "one: it sits below Pica Pica and can see the real "
+ "speaker signal and the real clock.\n\n"
+ "With PulseAudio or PipeWire it also means picking an "
+ "echo cancelling source as the microphone above - a "
+ "plain input device is not processed."));
+ }
+ else
+ {
+ rbEchoCancelPlatform->setToolTip(tr("Not available: the audio system in use does not "
+ "cancel echo for us."));
+
+ // A disabled button that is still the checked one would leave the
+ // group showing a choice that is not in effect.
+ if (rbEchoCancelPlatform->isChecked())
+ rbEchoCancelOwn->setChecked(true);
+ }
}
void SettingsDialog::fillAudioCaptureDevices()
@@ -1061,8 +1117,27 @@ void SettingsDialog::loadSettings()
if (audioPlaybackDevItem >= 0)
audioPlaybackDev->setCurrentIndex(audioPlaybackDevItem);
- cbEchoCancel->setChecked(st.loadValue("audio.echo_cancel", 1).toBool());
- cbAutoCaptureGain->setChecked(st.loadValue("audio.auto_capture_gain", 0).toBool());
+ // Not read from st directly: the setting has a default that depends on the
+ // platform, and an older on/off setting to fall back on.
+ switch (loadEchoCancellationSetting())
+ {
+ case EchoCancellationPlatform:
+ rbEchoCancelPlatform->setChecked(true);
+ break;
+
+ case EchoCancellationNone:
+ rbEchoCancelNone->setChecked(true);
+ break;
+
+ case EchoCancellationOwn:
+ default:
+ rbEchoCancelOwn->setChecked(true);
+ break;
+ }
+
+ // Covers a saved "platform" on a machine where it is no longer on offer -
+ // a Linux install whose audio system has since been moved to ALSA.
+ updateEchoCancellationChoices();
QString audioRingDevVal = st.loadValue("audio.ring_device", "default").toString();
int audioRingDevItem = audioRingDev->findData(audioRingDevVal);
@@ -1116,8 +1191,6 @@ void SettingsDialog::storeSettings()
st.storeValue("audio.capture_device", audioCaptureDev->itemData(audioCaptureDev->currentIndex()).toString());
st.storeValue("audio.playback_device", audioPlaybackDev->itemData(audioPlaybackDev->currentIndex()).toString());
st.storeValue("audio.ring_device", audioRingDev->itemData(audioRingDev->currentIndex()).toString());
- st.storeValue("audio.echo_cancel", cbEchoCancel->isChecked() ? "1" : "0");
- st.storeValue("audio.auto_capture_gain", cbAutoCaptureGain->isChecked() ? "1" : "0");
if (audioDriver)
{
@@ -1125,4 +1198,20 @@ void SettingsDialog::storeSettings()
st.storeValue("audio.driver", drv);
AudioDevice::SetLinuxDriverName(drv);
}
+
+ // After the driver, since on Linux that is what decides whether the
+ // platform option means anything - and updateEchoCancellationChoices()
+ // has already moved the selection off it if it does not.
+ EchoCancellationMode echoMode = EchoCancellationOwn;
+
+ if (rbEchoCancelPlatform->isChecked())
+ echoMode = EchoCancellationPlatform;
+ else if (rbEchoCancelNone->isChecked())
+ echoMode = EchoCancellationNone;
+
+ storeEchoCancellationSetting(echoMode);
+
+ // Publish it to the audio threads, the same way the driver above is
+ // published - see AudioDevice::SetEchoCancellation().
+ AudioDevice::SetEchoCancellation(echoMode);
}
diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h
index b879a74..ba04612 100644
--- a/pica-client/dialogs/settingsdialog.h
+++ b/pica-client/dialogs/settingsdialog.h
@@ -78,10 +78,18 @@ private:
// a slot hidden behind one ends up declared but never generated.
QComboBox *audioDriver;
- // Whether to run our own acoustic echo cancellation. Ignored when the
- // platform provides one - see AudioDevice::PlatformDriverName().
- QCheckBox *cbEchoCancel;
- QCheckBox *cbAutoCaptureGain;
+ // Who takes the echo of the other person back out of the microphone
+ // signal: EchoCanceller, the platform, or nobody. See
+ // EchoCancellationMode - one of the three, never two at once.
+ //
+ // The platform option is only offered where there is something to offer;
+ // on Linux that depends on the audio system chosen above it, so it is
+ // enabled and disabled as that combo box changes rather than once at
+ // startup.
+ QRadioButton *rbEchoCancelOwn;
+ QRadioButton *rbEchoCancelPlatform;
+ QRadioButton *rbEchoCancelNone;
+ void updateEchoCancellationChoices();
QComboBox *videoDev;
QPushButton *videoDevRefresh;
diff --git a/pica-client/echocanceller.cpp b/pica-client/echocanceller.cpp
index 2d7acf0..ad5b122 100644
--- a/pica-client/echocanceller.cpp
+++ b/pica-client/echocanceller.cpp
@@ -15,15 +15,47 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "echocanceller.h"
+#include "audiodevice.h"
+#include "settings.h"
+#include "globals.h"
#include <QDebug>
#include <QMutexLocker>
#include <cmath>
#include <cstring>
+// Which major version of webrtc-audio-processing this is being built against.
+// 1.x and 2.x are separate parallel-installable libraries with separate
+// pkg-config names, and 2.x broke the API in a few places - the conditionals
+// on this below are the whole of what differs as far as this file is
+// concerned.
+//
+// configure defines it, and so does the qmake build. The fallback is for
+// anything that builds this file without either: api/audio/audio_processing.h
+// is where 2.x moved the header that 1.x only had under modules/, so its
+// presence is the version.
+#ifndef PICA_WEBRTC_AUDIO_PROCESSING_MAJOR
+# if defined(__has_include) && __has_include(<api/audio/audio_processing.h>)
+# define PICA_WEBRTC_AUDIO_PROCESSING_MAJOR 2
+# else
+# define PICA_WEBRTC_AUDIO_PROCESSING_MAJOR 1
+# endif
+#endif
+
+// Still the right include on 2.x, where it is a stub forwarding to
+// api/audio/audio_processing.h - and it is installed there, so there is
+// nothing to switch on here.
+#include <modules/audio_processing/include/audio_processing.h>
+
+// Must come after the above, and not only as a matter of taste: on 2.x this
+// header declares parameters as absl::Nullable/absl::Nonnull without
+// including the abseil header that defines them, so on its own it does not
+// compile. audio_processing.h is what pulls that in. Both versions include
+// this themselves anyway; it is named here so that the scoped_refptr below is
+// not silently relying on somebody else's include.
+#include <api/scoped_refptr.h>
+
extern "C" {
-#include <speex/speex_echo.h>
-#include <speex/speex_preprocess.h>
#include <libswresample/swresample.h>
#include <libavutil/channel_layout.h>
#include <libavutil/samplefmt.h>
@@ -39,136 +71,217 @@ extern "C" {
# define EC_HAVE_CH_LAYOUT 0
#endif
-EchoCanceller::EchoCanceller(int sampleRate, int frameMs, int tailMs)
- : m_sampleRate(sampleRate > 0 ? sampleRate : 48000),
+// Setting values. Words rather than numbers because this is a three way
+// choice that will be read by whoever is looking at a user's config, and a
+// stray "2" says nothing.
+static const char kEchoCancellationOwn[] = "own";
+static const char kEchoCancellationPlatform[] = "platform";
+static const char kEchoCancellationNone[] = "off";
+
+EchoCancellationMode loadEchoCancellationSetting()
+{
+ Settings st(config_dbname);
+
+ const QString value = st.loadValue("audio.echo_cancellation", QString()).toString();
+
+ if (value == QLatin1String(kEchoCancellationOwn))
+ return EchoCancellationOwn;
+
+ if (value == QLatin1String(kEchoCancellationPlatform))
+ {
+ // The stored choice can stop being available underneath the user -
+ // the audio system moved from PulseAudio to ALSA, say. Leaving it as
+ // it stands would mean nobody cancelling and nothing saying so.
+ if (AudioDevice::PlatformEchoCancellationAvailable())
+ return EchoCancellationPlatform;
+
+ qWarning() << "Echo cancellation is set to the platform's own, but the audio system in "
+ "use does not provide one - falling back to ours";
+ return EchoCancellationOwn;
+ }
+
+ if (value == QLatin1String(kEchoCancellationNone))
+ return EchoCancellationNone;
+
+ // Nothing stored under the current key. An install that predates it may
+ // still have the on/off setting this replaced, and someone who turned
+ // cancellation off did mean it.
+ if (!st.loadValue("audio.echo_cancel", 1).toBool())
+ return EchoCancellationNone;
+
+ // The platform's own is the better one where it exists, so it is the
+ // default there; elsewhere the default is ours, which is what the plain
+ // on/off setting used to mean.
+ return AudioDevice::PlatformEchoCancellationAvailable()
+ ? EchoCancellationPlatform
+ : EchoCancellationOwn;
+}
+
+void storeEchoCancellationSetting(EchoCancellationMode mode)
+{
+ Settings st(config_dbname);
+
+ switch (mode)
+ {
+ case EchoCancellationPlatform:
+ st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationPlatform));
+ break;
+
+ case EchoCancellationNone:
+ st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationNone));
+ break;
+
+ case EchoCancellationOwn:
+ default:
+ st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationOwn));
+ break;
+ }
+}
+
+// Holds everything that needs the WebRTC headers, so that echocanceller.h -
+// which audiodevice.h includes, and so half the client with it - does not.
+struct EchoCancellerPrivate
+{
+ rtc::scoped_refptr<webrtc::AudioProcessing> apm;
+
+ // Both directions are mono at our rate. The integer interface requires
+ // the capture, playback and render rates to be the same, which is why
+ // pushFarEnd() resamples rather than passing the peer's rate through.
+ webrtc::StreamConfig stream;
+};
+
+bool EchoCanceller::isNativeRate(int rate)
+{
+ return rate == 8000 || rate == 16000 || rate == 32000 || rate == 48000;
+}
+
+EchoCanceller::EchoCanceller(int sampleRate)
+ : m_sampleRate(sampleRate),
m_frameSize(0),
- m_filterLength(0),
- m_echo(nullptr),
- m_preprocess(nullptr),
- m_farHead(0),
+ m_d(nullptr),
m_farSwr(nullptr),
m_farSwrRate(0),
- m_farCapacity(0),
m_nearEnergy(0.0),
- m_outEnergy(0.0),
m_farEnergy(0.0),
m_processedSamples(0),
- m_nextReportSamples(0)
+ m_nextReportSamples(0),
+ m_renderBlocks(0),
+ m_captureBlocks(0)
{
- if (frameMs <= 0)
- frameMs = 10;
- if (tailMs < frameMs)
- tailMs = frameMs;
-
- m_frameSize = (m_sampleRate * frameMs) / 1000;
- m_filterLength = (m_sampleRate * tailMs) / 1000;
-
- // speexdsp's MDF works in blocks of frame_size and wants the filter to
- // be a whole number of them.
- if (m_frameSize <= 0)
- m_frameSize = 480;
- m_filterLength = ((m_filterLength + m_frameSize - 1) / m_frameSize) * m_frameSize;
-
- // Room for the sound card's output buffer plus a healthy margin. See the
- // comment on m_farCapacity in the header for what overflowing it means.
- m_farCapacity = m_sampleRate; // one second
-
- SpeexEchoState *echo = speex_echo_state_init(m_frameSize, m_filterLength);
- if (!echo)
+ if (!isNativeRate(m_sampleRate))
{
- qWarning() << "Could not create speex echo canceller state";
+ qWarning() << "Echo canceller:" << m_sampleRate
+ << "Hz is not one of AEC3's rates (8/16/32/48 kHz) - not cancelling";
return;
}
- int rate = m_sampleRate;
- speex_echo_ctl(echo, SPEEX_ECHO_SET_SAMPLING_RATE, &rate);
+ // AEC3 works on 10ms of audio at a time, everywhere, at every rate.
+ m_frameSize = m_sampleRate / 100;
- SpeexPreprocessState *pre = speex_preprocess_state_init(m_frameSize, m_sampleRate);
- if (pre)
- {
- // The adaptive filter removes the part of the echo it can model; what
- // is left over - non-linearities in a cheap speaker, the tail of the
- // room past m_filterLength - has to be suppressed statistically, and
- // that is what the preprocessor's echo suppressor does with the
- // residual estimate the echo state hands it.
- speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_STATE, echo);
-
- int suppress = -40;
- speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS, &suppress);
- int suppressActive = -15;
- speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS_ACTIVE, &suppressActive);
-
- // Leave denoise on - it costs little next to the filter and keeps the
- // residual suppressor's noise estimate honest. AGC stays off: the
- // call has no level control anywhere else, and having one end quietly
- // ride the gain makes the echo path non-stationary, which is the one
- // thing an adaptive filter cannot follow.
- int on = 1, off = 0;
- speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_DENOISE, &on);
- speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_AGC, &off);
- speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_VAD, &off);
- }
- else
+ webrtc::AudioProcessing::Config cfg;
+
+ cfg.echo_canceller.enabled = true;
+ // AEC3 proper. The mobile mode is the cut down fixed point canceller for
+ // phones, and is worse anywhere that can afford not to use it.
+ cfg.echo_canceller.mobile_mode = false;
+ cfg.echo_canceller.enforce_high_pass_filtering = true;
+
+ cfg.high_pass_filter.enabled = true;
+
+ // Costs little next to the filter, and a lower noise floor is what lets
+ // the residual echo suppressor tell echo from room noise.
+ cfg.noise_suppression.enabled = true;
+ cfg.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate;
+
+ // Both AGCs stay off. The call has no level control anywhere else, and a
+ // gain riding the microphone moves the echo path, which is the one thing
+ // an adaptive filter cannot follow.
+ cfg.gain_controller1.enabled = false;
+ cfg.gain_controller2.enabled = false;
+
+ cfg.transient_suppression.enabled = false;
+
+#if PICA_WEBRTC_AUDIO_PROCESSING_MAJOR < 2
+ // All three were dropped in 2.x. The first two only gate statistics we
+ // never read; the third is a second, statistical echo estimator whose
+ // output we do not use either. Switching them off on 1.x is therefore
+ // the same configuration 2.x gives us for nothing.
+ cfg.voice_detection.enabled = false;
+ cfg.level_estimation.enabled = false;
+ cfg.residual_echo_detector.enabled = false;
+#endif
+
+ m_d = new EchoCancellerPrivate;
+ m_d->stream = webrtc::StreamConfig(m_sampleRate, 1);
+
+#if PICA_WEBRTC_AUDIO_PROCESSING_MAJOR >= 2
+ m_d->apm = webrtc::AudioProcessingBuilder().Create();
+#else
+ // 1.x hands back a raw pointer to an object whose reference count is
+ // still zero - the scoped_refptr is what takes it to one, and dropping
+ // the result on the floor would leak rather than free.
+ m_d->apm = rtc::scoped_refptr<webrtc::AudioProcessing>(webrtc::AudioProcessingBuilder().Create());
+#endif
+
+ if (!m_d->apm)
{
- qWarning() << "Could not create speex preprocessor - running without residual echo suppression";
+ qWarning() << "Could not create the WebRTC audio processing module";
+ delete m_d;
+ m_d = nullptr;
+ return;
}
- m_echo = reinterpret_cast<SpeexEchoState_ *>(echo);
- m_preprocess = reinterpret_cast<SpeexPreprocessState_ *>(pre);
+ m_d->apm->ApplyConfig(cfg);
- m_farBlock.resize(m_frameSize);
- m_farFifo.reserve(m_farCapacity);
+ // Mono at our rate in both directions, which the integer interface
+ // requires anyway. Spelled out through a ProcessingConfig rather than the
+ // shorter Initialize() overload taking channel layouts, because that
+ // overload and the ChannelLayout enum it needs are gone in 2.x.
+ webrtc::ProcessingConfig streams;
+ streams.input_stream() = m_d->stream;
+ streams.output_stream() = m_d->stream;
+ streams.reverse_input_stream() = m_d->stream;
+ streams.reverse_output_stream() = m_d->stream;
+
+ const int err = m_d->apm->Initialize(streams);
+ if (err != webrtc::AudioProcessing::kNoError)
+ {
+ qWarning() << "Could not initialise the WebRTC audio processing module:" << err;
+ delete m_d;
+ m_d = nullptr;
+ return;
+ }
// Hold off the first report until there is a window's worth of audio
// behind the averages, otherwise it only says the call just started.
m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec;
- qDebug() << "Echo canceller:" << m_sampleRate << "Hz, frame" << m_frameSize
- << "samples, tail" << m_filterLength << "samples ("
- << (m_filterLength * 1000 / m_sampleRate) << "ms )";
+ // The library version is in there because it is otherwise invisible which
+ // of the two parallel-installable ones got linked, and they behave
+ // differently enough to be worth knowing about from a log alone.
+ qDebug("Echo canceller: WebRTC AEC3 (webrtc-audio-processing-%d), %d Hz, frame %d samples",
+ PICA_WEBRTC_AUDIO_PROCESSING_MAJOR, m_sampleRate, m_frameSize);
}
EchoCanceller::~EchoCanceller()
{
- // Order matters: the preprocessor holds a pointer to the echo state.
- if (m_preprocess)
- speex_preprocess_state_destroy(reinterpret_cast<SpeexPreprocessState *>(m_preprocess));
- if (m_echo)
- speex_echo_state_destroy(reinterpret_cast<SpeexEchoState *>(m_echo));
+ delete m_d;
+
if (m_farSwr)
swr_free(&m_farSwr);
}
-void EchoCanceller::reset()
+bool EchoCanceller::isValid() const
{
- {
- QMutexLocker locker(&m_procMutex);
- if (m_echo)
- speex_echo_state_reset(reinterpret_cast<SpeexEchoState *>(m_echo));
- }
-
- {
- QMutexLocker locker(&m_farMutex);
- m_farFifo.clear();
- m_farHead = 0;
- }
-
- {
- QMutexLocker locker(&m_statMutex);
- m_nearEnergy = 0.0;
- m_outEnergy = 0.0;
- m_farEnergy = 0.0;
- m_processedSamples = 0;
- m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec;
- }
+ return m_d != nullptr;
}
void EchoCanceller::pushFarEnd(const qint16 *pcm, int nsamples, int srcRate)
{
- if (!m_echo || !pcm || nsamples <= 0)
+ if (!m_d || !pcm || nsamples <= 0)
return;
- QMutexLocker locker(&m_farMutex);
+ QMutexLocker locker(&m_renderMutex);
if (srcRate > 0 && srcRate != m_sampleRate)
{
@@ -226,97 +339,59 @@ void EchoCanceller::pushFarEnd(const qint16 *pcm, int nsamples, int srcRate)
nsamples = got;
}
- // Reclaim the space already consumed rather than growing without bound;
- // doing it only once the head has run past half the buffer keeps this
- // from turning into a memmove on every call.
- if (m_farHead > 0 && m_farHead >= m_farFifo.size() / 2)
- {
- m_farFifo.remove(0, m_farHead);
- m_farHead = 0;
- }
+ int base = m_farPending.size();
+ m_farPending.resize(base + nsamples);
+ memcpy(m_farPending.data() + base, pcm, nsamples * sizeof(qint16));
- int oldSize = m_farFifo.size();
- m_farFifo.resize(oldSize + nsamples);
- memcpy(m_farFifo.data() + oldSize, pcm, nsamples * sizeof(qint16));
+ double farEnergy = 0.0;
+ int blocks = m_farPending.size() / m_frameSize;
- int pending = m_farFifo.size() - m_farHead;
- if (pending > m_farCapacity)
+ for (int b = 0; b < blocks; b++)
{
- m_farHead += pending - m_farCapacity;
- if (m_farHead >= m_farFifo.size())
- {
- m_farFifo.clear();
- m_farHead = 0;
- }
- }
-}
+ qint16 *farBlock = m_farPending.data() + b * m_frameSize;
-bool EchoCanceller::popFarEnd(qint16 *dst, int nsamples)
-{
- QMutexLocker locker(&m_farMutex);
-
- int pending = m_farFifo.size() - m_farHead;
- if (pending <= 0)
- return false;
+ for (int i = 0; i < m_frameSize; i++)
+ farEnergy += double(farBlock[i]) * double(farBlock[i]);
- int n = pending < nsamples ? pending : nsamples;
- memcpy(dst, m_farFifo.constData() + m_farHead, n * sizeof(qint16));
- m_farHead += n;
+ // In place: the module is happy for source and destination to be the
+ // same buffer, and nothing downstream wants the processed render
+ // signal - handing it over is only how AEC3 learns what is about to
+ // come back through the microphone.
+ m_d->apm->ProcessReverseStream(farBlock, m_d->stream, m_d->stream, farBlock);
+ }
- if (n < nsamples)
- memset(dst + n, 0, (nsamples - n) * sizeof(qint16));
+ if (blocks > 0)
+ {
+ m_farPending.remove(0, blocks * m_frameSize);
- return true;
+ QMutexLocker statLocker(&m_statMutex);
+ m_farEnergy = m_farEnergy * 0.95 + (farEnergy / (blocks * m_frameSize)) * 0.05;
+ m_renderBlocks += blocks;
+ }
}
void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples)
{
- if (!m_echo || !pcm || nsamples < m_frameSize)
+ if (!m_d || !pcm || nsamples < m_frameSize)
return;
- QMutexLocker locker(&m_procMutex);
-
- SpeexEchoState *echo = reinterpret_cast<SpeexEchoState *>(m_echo);
- SpeexPreprocessState *pre = reinterpret_cast<SpeexPreprocessState *>(m_preprocess);
-
double nearEnergy = 0.0;
- double outEnergy = 0.0;
- double farEnergy = 0.0;
-
int blocks = nsamples / m_frameSize;
- for (int b = 0; b < blocks; b++)
- {
- // Not called "near" - windef.h still defines that as an empty macro
- // on the mingw-w64 builds.
- qint16 *nearBlock = pcm + b * m_frameSize;
- for (int i = 0; i < m_frameSize; i++)
- nearEnergy += double(nearBlock[i]) * double(nearBlock[i]);
+ {
+ QMutexLocker locker(&m_captureMutex);
- if (!popFarEnd(m_farBlock.data(), m_frameSize))
+ for (int b = 0; b < blocks; b++)
{
- // Nothing is being played, so there is no echo to remove. Still
- // run the frame through, otherwise the filter's idea of the far
- // end signal silently runs ahead of the microphone.
- memset(m_farBlock.data(), 0, m_frameSize * sizeof(qint16));
- }
-
- // Measured before cancellation - this is the reference the filter is
- // about to be given, so it says whether there was anything coming out
- // of the speaker for it to work on.
- for (int i = 0; i < m_frameSize; i++)
- farEnergy += double(m_farBlock[i]) * double(m_farBlock[i]);
-
- // speex_echo_cancellation() is happy to work in place on the near end
- // buffer, which is what lets this sit directly in the capture path
- // without another copy.
- speex_echo_cancellation(echo, nearBlock, m_farBlock.constData(), nearBlock);
+ // Not called "near" - windef.h still defines that as an empty
+ // macro on the mingw-w64 builds.
+ qint16 *nearBlock = pcm + b * m_frameSize;
- if (pre)
- speex_preprocess_run(pre, nearBlock);
+ for (int i = 0; i < m_frameSize; i++)
+ nearEnergy += double(nearBlock[i]) * double(nearBlock[i]);
- for (int i = 0; i < m_frameSize; i++)
- outEnergy += double(nearBlock[i]) * double(nearBlock[i]);
+ m_d->apm->ProcessStream(nearBlock, m_d->stream, m_d->stream, nearBlock);
+ }
}
// At least one whole block, guaranteed by the nsamples check on entry.
@@ -326,14 +401,13 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples)
{
QMutexLocker statLocker(&m_statMutex);
- // Exponential forgetting, so erle() reflects the last few seconds
+ // Exponential forgetting, so the levels reflect the last few seconds
// rather than the whole call. Per sample, so a change in the capture
- // path's block size does not move the levels.
+ // path's block size does not move them.
m_nearEnergy = m_nearEnergy * 0.95 + (nearEnergy / used) * 0.05;
- m_outEnergy = m_outEnergy * 0.95 + (outEnergy / used) * 0.05;
- m_farEnergy = m_farEnergy * 0.95 + (farEnergy / used) * 0.05;
m_processedSamples += used;
+ m_captureBlocks += blocks;
if (m_processedSamples >= m_nextReportSamples)
{
m_nextReportSamples = m_processedSamples + qint64(m_sampleRate) * kReportIntervalSec;
@@ -347,23 +421,73 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples)
// does not matter for a log line.
if (report)
{
- int depth = farEndDepth();
+ int delayMs = 0;
+ double erleDb = 0.0;
+ double erlDb = 0.0;
+ bool haveErl = false;
+
+ {
+ QMutexLocker locker(&m_captureMutex);
+ webrtc::AudioProcessingStats stats = m_d->apm->GetStatistics();
+
+ if (stats.delay_ms.has_value())
+ delayMs = *stats.delay_ms;
+ if (stats.echo_return_loss_enhancement.has_value())
+ erleDb = *stats.echo_return_loss_enhancement;
+ if (stats.echo_return_loss.has_value())
+ {
+ erlDb = *stats.echo_return_loss;
+ haveErl = true;
+ }
+ }
+
+ qint64 lead = 0;
+ {
+ QMutexLocker statLocker(&m_statMutex);
+ lead = m_renderBlocks - m_captureBlocks;
+ }
- qDebug("AEC: erle %+.1f dB mic %.1f dBFS far %.1f dBFS ref backlog %d ms%s",
- erle(), nearEndLevelDb(), farEndLevelDb(),
- (depth * 1000) / m_sampleRate,
- depth >= m_farCapacity ? " [FIFO FULL - reference drifting]" : "");
+ // erl says how strongly the speaker couples back into the microphone,
+ // which is whether there is anything worth cancelling at all. AEC3
+ // reports it negated (echo_remover.cc: -10*log10(erl)), so the more
+ // negative it is the weaker the coupling - a headset reads well down,
+ // speakers in a small room read close to zero.
+ //
+ // erle is how much of that coupling the filter is actually removing.
+ // It sits at its 0 dB floor (Erle::min = 1.f) until the filter
+ // converges, so a call that never leaves +0.2 dB is a call where the
+ // linear filter never got anywhere.
+ //
+ // The delay is what AEC3 has aligned the two streams at. Zero is not
+ // a delay: block_processor.cc reports 0 when the estimator has no
+ // value, so it means the filter is running unaligned and the erle
+ // beside it cannot be anything but the floor.
+ QString delayText = delayMs > 0
+ ? QString("echo delay %1 ms").arg(delayMs)
+ : QStringLiteral("NOT ALIGNED");
+
+ QString erlText = haveErl
+ ? QString("erl %1 dB").arg(erlDb, 0, 'f', 1)
+ : QStringLiteral("erl n/a");
+
+ qDebug("AEC: erle %+.1f dB %s mic %.1f dBFS far %.1f dBFS %s render lead %d ms",
+ erleDb, qPrintable(erlText), nearEndLevelDb(), farEndLevelDb(),
+ qPrintable(delayText), (int)(lead * 10)); // a block is 10ms, always
}
}
double EchoCanceller::erle()
{
- QMutexLocker locker(&m_statMutex);
-
- if (m_nearEnergy <= 0.0 || m_outEnergy <= 0.0)
+ if (!m_d)
return 0.0;
- return 10.0 * log10(m_nearEnergy / m_outEnergy);
+ QMutexLocker locker(&m_captureMutex);
+
+ webrtc::AudioProcessingStats stats = m_d->apm->GetStatistics();
+
+ return stats.echo_return_loss_enhancement.has_value()
+ ? *stats.echo_return_loss_enhancement
+ : 0.0;
}
// Mean square of 16 bit samples to dBFS, with a floor so digital silence
@@ -390,11 +514,3 @@ double EchoCanceller::farEndLevelDb()
QMutexLocker locker(&m_statMutex);
return level_db(m_farEnergy);
}
-
-int EchoCanceller::farEndDepth()
-{
- QMutexLocker locker(&m_farMutex);
-
- int pending = m_farFifo.size() - m_farHead;
- return pending > 0 ? pending : 0;
-}
diff --git a/pica-client/echocanceller.h b/pica-client/echocanceller.h
index e59f8d7..faf50dd 100644
--- a/pica-client/echocanceller.h
+++ b/pica-client/echocanceller.h
@@ -22,60 +22,79 @@
#include <QVector>
#include <QSharedPointer>
-struct SpeexEchoState_;
-struct SpeexPreprocessState_;
+struct EchoCancellerPrivate;
struct SwrContext;
-// Acoustic echo cancellation for a call, using speexdsp's frequency domain
-// adaptive filter (MDF).
+// Where the acoustic echo of a call is taken out of the microphone signal.
+// One of these three, never two at once: running a second canceller over an
+// already cancelled signal gives it a near silent reference to chase and it
+// chews holes in the speech.
+enum EchoCancellationMode
+{
+ // EchoCanceller below - WebRTC's AEC3, in process.
+ EchoCancellationOwn,
+
+ // Whatever the operating system or sound server does below us, which is
+ // better where it exists: it sees the real speaker signal and the real
+ // clock. Not offered everywhere - see
+ // AudioDevice::PlatformEchoCancellationAvailable().
+ EchoCancellationPlatform,
+
+ // Nothing cancels. Correct with a headset, and the only honest answer
+ // when both of the above make things worse.
+ EchoCancellationNone
+};
+
+// The "audio.echo_cancellation" setting. Both must be called from a thread
+// that may touch the settings database - see AudioDevice::SetLinuxDriverName()
+// for why the audio threads may not.
+//
+// Nothing stored yet means the default for this platform: the platform's own
+// where there is one, ours otherwise. The setting this replaced
+// ("audio.echo_cancel", a plain on/off) is still honoured when it is all
+// there is, so an existing install that had it switched off stays that way.
+EchoCancellationMode loadEchoCancellationSetting();
+void storeEchoCancellationSetting(EchoCancellationMode mode);
+
+// Acoustic echo cancellation for a call, using WebRTC's AEC3 through the
+// audio processing module.
//
// One instance is shared by the two AudioDevice objects of a call - the
// playback one feeds it the far end signal it is about to hand to the sound
// card via pushFarEnd(), the capture one runs the microphone signal through
-// processNearEnd() before encoding. Both run on their own thread, so every
-// entry point here takes a lock.
+// processNearEnd() before encoding. Both run on their own thread, which is
+// what AEC3 expects: one render thread, one capture thread.
//
// Because AudioVideoCallController tears a call down without joining those
// two threads (see stopAudioPipeline()), ownership is shared: hold this
// through a QSharedPointer so the object outlives whichever thread leaves
// last.
//
-// Alignment between the two signals is left to the FIFO: in the steady state
-// the playback side pushes samples at the same rate the capture side pulls
-// them, and the backlog that builds up in the FIFO settles at exactly the
-// sound card's buffer depth - which is also how far ahead of the speaker the
-// playback side is running. What remains is the acoustic flight time from
-// speaker to microphone plus any capture side buffering, and that is what the
-// filter tail is for.
+// One per call, built when the call starts and dropped when it ends. There is
+// nothing worth carrying over: the previous call's room response is not this
+// call's, and the module is cheap enough to build.
+//
+// Nothing here tries to line the two signals up. AEC3 has its own delay
+// estimator and a render buffer to hold the far end audio until the echo of
+// it arrives, which is the whole reason it copes with an application that
+// hands over playback audio some way ahead of the speaker actually making a
+// sound - and with that lead changing when the sound card's buffer does.
class EchoCanceller
{
public:
- // frameMs is the block the filter works on and must divide the audio
- // handed to processNearEnd().
- //
- // tailMs is the longest echo delay the filter can model at all; anything
- // arriving later is invisible to it and can only be dealt with
- // statistically by the residual suppressor. The fixed part of that delay
- // is small - the capture path plus a metre or two of flight, call it 30ms
- // - so nearly all of the tail is budget for the room's reverberation.
- //
- // It is not free in either direction. Too short and late reflections never
- // get modelled and are heard as residual echo; too long and both the
- // convergence time and the steady state misadjustment grow with the tap
- // count. 300ms at 16kHz is 4800 taps, which is still fewer than the 7200
- // this ran at when calls were 48kHz with a 150ms tail - so the room
- // coverage doubles and the filter is still shorter than it used to be.
- EchoCanceller(int sampleRate, int frameMs = 10, int tailMs = 300);
+ // sampleRate must be one of AEC3's native rates - 8000, 16000, 32000 or
+ // 48000. Anything else leaves isValid() false rather than silently
+ // processing at the wrong rate; the caller then runs without
+ // cancellation.
+ explicit EchoCanceller(int sampleRate);
~EchoCanceller();
- bool isValid() const { return m_echo != nullptr; }
+ bool isValid() const;
int sampleRate() const { return m_sampleRate; }
- int frameSize() const { return m_frameSize; }
- // Forget the adapted filter and drop any buffered far end audio. Called
- // when a call starts, since the previous call's room response is not
- // worth keeping and a stale FIFO would start it off misaligned.
- void reset();
+ // The block AEC3 works on: 10ms, always. Audio handed to
+ // processNearEnd() should be a whole number of these.
+ int frameSize() const { return m_frameSize; }
// Playback side: the audio about to be written to the output device.
//
@@ -83,9 +102,7 @@ public:
// we capture at a rate of our choosing but play back at whatever the peer
// announced, so a call with a build that negotiates a different rate ends
// up with the two directions disagreeing. Passing 0 means "same as ours".
- // Anything else is resampled on the way into the FIFO - without this the
- // playback side used to drop the canceller entirely on a rate mismatch,
- // leaving the capture side cancelling against silence.
+ // Anything else is resampled on the way in.
void pushFarEnd(const qint16 *pcm, int nsamples, int srcRate = 0);
// Capture side: removes the echo of what pushFarEnd() was given, in
@@ -93,74 +110,57 @@ public:
// partial trailing block is passed through untouched.
void processNearEnd(qint16 *pcm, int nsamples);
- // Rough measure of how well it is working, in dB - how much quieter the
- // output is than the microphone input while the far end is active. Zero
- // until enough far end audio has gone through to mean anything.
+ // Echo return loss enhancement in dB, as AEC3 measures it: how much
+ // quieter the echo is after cancellation than before. Zero until the
+ // filter has had far end audio to work on.
//
// Only meaningful while the far end is actually talking, which is what
// farEndLevelDb() is for: with nothing coming out of the speaker there is
// no echo to remove and the figure means nothing.
double erle();
- // Recent signal levels in dBFS, averaged over the same window as erle().
+ // Recent signal levels in dBFS, averaged over a few seconds.
// nearEndLevelDb() is the microphone before cancellation - a floor-level
// reading there means the capture path is dead and nothing else in the
// numbers is worth reading.
double nearEndLevelDb();
double farEndLevelDb();
- // Far end samples waiting to be paired with microphone audio. In the
- // steady state this settles at the depth of the sound card's output
- // buffer and stays there, because both are drained by the same clock.
- // Climbing towards m_farCapacity means capture and playback are running
- // off different clocks - separate cards - and the reference is drifting
- // out of alignment with the echo it is supposed to explain.
- int farEndDepth();
-
private:
Q_DISABLE_COPY(EchoCanceller)
int m_sampleRate;
int m_frameSize;
- int m_filterLength;
- // speexdsp state. The echo canceller proper, plus a preprocessor whose
- // only job here is the residual echo suppression that the adaptive
- // filter alone cannot do (SPEEX_PREPROCESS_SET_ECHO_STATE).
- SpeexEchoState_ *m_echo;
- SpeexPreprocessState_ *m_preprocess;
+ // The audio processing module and the stream configurations handed to it,
+ // kept out of this header so that the WebRTC headers - which want C++17
+ // and drag in abseil - are only seen by echocanceller.cpp.
+ EchoCancellerPrivate *m_d;
- // Guards the speex state, which both directions reach into: the capture
- // thread through speex_echo_cancellation(), and reset() from whichever
- // thread starts the call.
- QMutex m_procMutex;
+ // Guards the render direction: the far end accumulator, the resampler,
+ // and ProcessReverseStream(). Only the playback thread takes it.
+ QMutex m_renderMutex;
- // Far end FIFO, written by the playback thread and drained by the
- // capture thread.
- QMutex m_farMutex;
- QVector<qint16> m_farFifo;
- int m_farHead;
+ // Guards the capture direction: ProcessStream() and the statistics read
+ // alongside it. Taken by the capture thread, and by whoever calls the
+ // erle() accessor.
+ QMutex m_captureMutex;
+
+ // Far end audio that did not fill a whole 10ms block, waiting for the
+ // samples that will complete it.
+ QVector<qint16> m_farPending;
// Rate conversion for a far end that is not running at our rate, built on
- // first use and rebuilt if the rate changes. Only the playback thread
- // touches these, under m_farMutex.
+ // first use and rebuilt if the rate changes.
SwrContext *m_farSwr;
int m_farSwrRate;
QVector<qint16> m_farScratch;
- // Upper bound on the FIFO, in samples. Reached only if the two sound
- // cards' clocks drift apart or playback outruns capture; past this the
- // oldest audio is dropped, since keeping it would mean the reference
- // signal lagging further and further behind the echo it is meant to
- // explain.
- int m_farCapacity;
-
- // Exponentially averaged mean square of the near end (microphone), the
- // cancelled output and the far end reference, for erle() and the level
- // readings. Per sample rather than per call, so they stay comparable
- // whatever block size the capture path hands us.
+ // Exponentially averaged mean square of the near end (microphone) and of
+ // the far end reference, for the level readings. Per sample rather than
+ // per call, so they stay comparable whatever block size the capture path
+ // hands us.
double m_nearEnergy;
- double m_outEnergy;
double m_farEnergy;
QMutex m_statMutex;
@@ -170,13 +170,21 @@ private:
qint64 m_processedSamples;
qint64 m_nextReportSamples;
+ // 10ms blocks handed to each direction. AEC3 lines the two streams up by
+ // counting API calls, not by any clock, so the gap between these is what
+ // its alignment is measured in. In a healthy call it settles at the depth
+ // of the sound card's output buffer and stays there; a gap that keeps
+ // growing or shrinking means capture and playback are running off
+ // different clocks and the alignment is being dragged out from under the
+ // filter. Guarded by m_statMutex.
+ qint64 m_renderBlocks;
+ qint64 m_captureBlocks;
+
// How often the canceller reports how it is doing, in seconds.
static const int kReportIntervalSec = 2;
- // Scratch far end block, reused by processNearEnd().
- QVector<qint16> m_farBlock;
-
- bool popFarEnd(qint16 *dst, int nsamples);
+ // True if rate is one of the rates AEC3's integer interface accepts.
+ static bool isNativeRate(int rate);
};
typedef QSharedPointer<EchoCanceller> EchoCancellerPtr;
diff --git a/pica-client/main.cpp b/pica-client/main.cpp
index 330f487..3972ab8 100644
--- a/pica-client/main.cpp
+++ b/pica-client/main.cpp
@@ -449,14 +449,21 @@ static bool create_config_dir()
QFile::setPermissions(config_dbname, QFile::ReadOwner | QFile::WriteOwner);
- // Publish the audio driver choice to the audio threads while we are still
- // the only thread and the settings database is reachable - see
- // AudioDevice::SetLinuxDriverName() for why it cannot be read on demand.
+ // Publish the audio driver choice and the echo cancellation setting to the
+ // audio threads while we are still the only thread and the settings
+ // database is reachable - see AudioDevice::SetLinuxDriverName() for why
+ // they cannot be read on demand.
+ //
+ // The driver first: on Linux it is what decides whether the platform has
+ // any cancellation to offer, which loadEchoCancellationSetting() needs in
+ // order to work out the default.
{
Settings audiost(config_dbname);
AudioDevice::SetLinuxDriverName(audiost.loadValue("audio.driver", "alsa").toString());
}
+ AudioDevice::SetEchoCancellation(loadEchoCancellationSetting());
+
/*if (!QFile::exists(config_dir + QDir::separator() + PICA_CLIENT_DHPARAMFILE))
{
OpenSSLTool osslt;
diff --git a/pica-client/nativeaudio.h b/pica-client/nativeaudio.h
index fa3cccb..3e364c6 100644
--- a/pica-client/nativeaudio.h
+++ b/pica-client/nativeaudio.h
@@ -43,14 +43,22 @@
// What a stream is going to be used for.
//
-// A call stream asks the platform for its voice processing, echo cancellation
-// above all. A plain one does not, and must not: on macOS the echo canceller
-// is an audio unit that captures as well as renders, so asking for it to play
-// a ringtone would open the microphone, prompt for permission and light the
-// recording indicator, all to make a telephone noise.
+// The two call modes differ in one thing only: whether the platform is asked
+// for its voice processing, echo cancellation above all. Which of them a call
+// uses is the "audio.echo_cancellation" setting - see EchoCancellationMode.
+// They agree on everything else, device selection included: a call belongs on
+// the endpoint the user nominated for calls whoever ends up cancelling the
+// echo.
+//
+// A plain stream is not a call at all, and must not ask for voice processing:
+// on macOS the echo canceller is an audio unit that captures as well as
+// renders, so asking for it to play a ringtone would open the microphone,
+// prompt for permission and light the recording indicator, all to make a
+// telephone noise.
enum NativeAudioMode
{
NativeAudioVoiceCall,
+ NativeAudioVoiceCallRaw,
NativeAudioPlain
};
diff --git a/pica-client/nativeaudio_coreaudio.cpp b/pica-client/nativeaudio_coreaudio.cpp
index be0d604..6dc9ad7 100644
--- a/pica-client/nativeaudio_coreaudio.cpp
+++ b/pica-client/nativeaudio_coreaudio.cpp
@@ -147,6 +147,30 @@ static QVector<AudioDeviceID> allDevices()
return devices;
}
+// The system's current default input or output device.
+//
+// Needed because kAudioUnitSubType_HALOutput left alone follows the default
+// OUTPUT device on both of its buses, so an input only unit that is never
+// bound to a device tries to capture from the speakers. VoiceProcessingIO has
+// no such problem, which is why nothing needed this until there was a plain
+// capture path.
+static AudioDeviceID defaultDevice(bool input)
+{
+ AudioObjectPropertyAddress addr;
+ addr.mSelector = input ? kAudioHardwarePropertyDefaultInputDevice
+ : kAudioHardwarePropertyDefaultOutputDevice;
+ addr.mScope = kAudioObjectPropertyScopeGlobal;
+ addr.mElement = kAudioObjectPropertyElementMaster;
+
+ AudioDeviceID dev = kAudioObjectUnknown;
+ UInt32 size = sizeof(dev);
+
+ if (AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, 0, nullptr, &size, &dev) != noErr)
+ return kAudioObjectUnknown;
+
+ return dev;
+}
+
// Looks up a device by the UID string enumerate() handed out.
static AudioDeviceID deviceForUid(const QString &uid, bool input)
{
@@ -174,10 +198,12 @@ static AudioDeviceID deviceForUid(const QString &uid, bool input)
// through the same unit, and AudioDevice keeps one object per direction on its
// own thread, the voice unit is a refcounted singleton the two of them share.
//
-// A plain unit is kAudioUnitSubType_HALOutput, output only, no voice
-// processing, one per caller. Ringtones and earpiece tones use it: they are
-// not part of a call, and opening a voice unit to play one would switch the
-// microphone on, prompt for permission and light the recording indicator.
+// A plain unit is kAudioUnitSubType_HALOutput with no voice processing, one
+// per caller, running one direction only. Ringtones and earpiece tones use
+// the output flavour: they are not part of a call, and opening a voice unit
+// to play one would switch the microphone on, prompt for permission and light
+// the recording indicator. The input flavour is for a call that cancels its
+// own echo, or none of it - see NativeAudioVoiceCallRaw.
//
// One consequence of the singleton worth knowing about: an audio unit is
// bound to one device, so capture and playback cannot be on different sound
@@ -196,6 +222,12 @@ public:
static CoreAudioUnit *createPlain(const QString &device, int sampleRate, int channels,
QString *error);
+ // The same, input only: a microphone opened without the voice unit, for a
+ // call whose echo is cancelled somewhere else - by EchoCanceller, or not
+ // at all. Owned by the caller like createPlain().
+ static CoreAudioUnit *createPlainCapture(const QString &device, int sampleRate, int channels,
+ QString *error);
+
void release(bool forCapture);
AudioRing captureRing;
@@ -312,25 +344,41 @@ bool CoreAudioUnit::setUp(const QString &device, QString *error)
// Bind to a specific device when one was asked for. Left alone, the unit
// follows the system default input and output, which is what most users
// want and what keeps working when they unplug a headset mid-call.
+ //
+ // With one exception: a HAL unit follows the default OUTPUT device on
+ // both of its buses, so an input only one has to be pointed at the
+ // default input by hand or it captures from nothing.
+ AudioDeviceID devId = kAudioObjectUnknown;
+
if (!device.isEmpty() && device != QLatin1String("default"))
{
- AudioDeviceID devId = deviceForUid(device, m_wantInput);
+ devId = deviceForUid(device, m_wantInput);
if (devId == kAudioObjectUnknown)
devId = deviceForUid(device, !m_wantInput);
- if (devId != kAudioObjectUnknown)
- {
- err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_CurrentDevice,
- kAudioUnitScope_Global, kOutputBus, &devId, sizeof(devId));
- if (err != noErr)
- qWarning() << "CoreAudio: could not bind the audio unit to device" << device
- << osStatusString(err) << "- using the system default";
- }
- else
- {
+ if (devId == kAudioObjectUnknown)
qWarning() << "CoreAudio: device" << device << "not found - using the system default";
+ }
+
+ if (devId == kAudioObjectUnknown && m_wantInput && !m_wantOutput)
+ {
+ devId = defaultDevice(true);
+ if (devId == kAudioObjectUnknown)
+ {
+ if (error)
+ *error = QStringLiteral("There is no default audio input device");
+ return false;
}
}
+
+ if (devId != kAudioObjectUnknown)
+ {
+ err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_CurrentDevice,
+ kAudioUnitScope_Global, kOutputBus, &devId, sizeof(devId));
+ if (err != noErr)
+ qWarning() << "CoreAudio: could not bind the audio unit to device" << device
+ << osStatusString(err) << "- using the system default";
+ }
m_device = device;
UInt32 maxFrames = kMaxFramesPerSlice;
@@ -390,7 +438,7 @@ bool CoreAudioUnit::setUp(const QString &device, QString *error)
kAudioUnitScope_Global, 0, &bypass, sizeof(bypass));
// Leave the unit's automatic gain control off, for the same reason
- // the speexdsp path leaves speex's off: a moving gain in front of a
+ // EchoCanceller leaves WebRTC's off: a moving gain in front of a
// call that has no level control anywhere else is more trouble than
// it is worth.
UInt32 agc = 0;
@@ -616,6 +664,20 @@ CoreAudioUnit *CoreAudioUnit::createPlain(const QString &device, int sampleRate,
return unit;
}
+CoreAudioUnit *CoreAudioUnit::createPlainCapture(const QString &device, int sampleRate, int channels,
+ QString *error)
+{
+ CoreAudioUnit *unit = new CoreAudioUnit(sampleRate, channels, false, true, false);
+ if (!unit->setUp(device, error))
+ {
+ delete unit;
+ return nullptr;
+ }
+
+ unit->m_captureRefs = 1;
+ return unit;
+}
+
void CoreAudioUnit::release(bool forCapture)
{
QMutexLocker locker(&s_mutex);
@@ -668,15 +730,18 @@ public:
{
close();
- // There is no plain capture path: the only reason this backend
- // captures at all is the call, and the call always wants the voice
- // unit.
- Q_UNUSED(mode)
+ // The voice unit is the whole point of this backend, but it is not
+ // the only thing it can open: a call that cancels its own echo wants
+ // the microphone as the hardware delivers it, with nothing having
+ // been subtracted from it first.
+ if (mode == NativeAudioVoiceCall)
+ m_unit = CoreAudioUnit::acquireVoice(device, sampleRate, channels, true, error);
+ else
+ m_unit = CoreAudioUnit::createPlainCapture(device, sampleRate, channels, error);
- m_unit = CoreAudioUnit::acquireVoice(device, sampleRate, channels, true, error);
m_isCapture = true;
m_channels = channels;
- m_platformAec = (m_unit != nullptr);
+ m_platformAec = (m_unit != nullptr && m_unit->voiceProcessing());
return m_unit != nullptr;
}
diff --git a/pica-client/nativeaudio_wasapi.cpp b/pica-client/nativeaudio_wasapi.cpp
index 449dcd7..c0f5130 100644
--- a/pica-client/nativeaudio_wasapi.cpp
+++ b/pica-client/nativeaudio_wasapi.cpp
@@ -299,7 +299,11 @@ IMMDevice *WasapiBackend::resolveDevice(const QString &device, bool isCapture,
// A ringtone is not a call and belongs on the console endpoint, so
// that it is heard on the speakers rather than in a headset the user
// is not wearing yet.
- const ERole role = (mode == NativeAudioVoiceCall) ? eCommunications : eConsole;
+ //
+ // Both call modes take the communications endpoint. Cancelling the
+ // echo ourselves is a reason not to ask for the endpoint's voice
+ // processing, not a reason to move the call to a different device.
+ const ERole role = (mode != NativeAudioPlain) ? eCommunications : eConsole;
hr = devEnum->GetDefaultAudioEndpoint(isCapture ? eCapture : eRender, role, &dev);
}
else
@@ -407,6 +411,10 @@ bool WasapiBackend::open(const QString &device, int sampleRate, int channels, bo
// Asking for it on render as well is what made the settings dialog's
// microphone test come out chopped while the ring test, which opens a
// plain stream, was clean.
+ //
+ // NativeAudioVoiceCallRaw is a call that cancels its own echo, so it does
+ // not want the endpoint's processing either - this is the one place where
+ // it differs from NativeAudioVoiceCall.
if (mode == NativeAudioVoiceCall && isCapture)
requestCommunicationsCategory();
diff --git a/pica-client/pica-client.pro b/pica-client/pica-client.pro
index 9dcc1ab..ddbcda8 100644
--- a/pica-client/pica-client.pro
+++ b/pica-client/pica-client.pro
@@ -16,9 +16,6 @@ SOURCES += main.cpp\
nativeaudio.cpp \
nativeaudio_wasapi.cpp \
nativeaudio_coreaudio.cpp \
- capturegain.cpp \
- capturegain_wasapi.cpp \
- capturegain_coreaudio.cpp \
audiovideocallcontroller.cpp \
mainwindow.cpp \
chatwindow.cpp \
@@ -62,7 +59,6 @@ HEADERS += mainwindow.h \
audioring.h \
echocanceller.h \
nativeaudio.h \
- capturegain.h \
audiovideocallcontroller.h \
chatwindow.h \
callwindow.h \
@@ -119,11 +115,34 @@ unix: PKGCONFIG += libcrypto
unix|win32: LIBS += -lminiupnpc
-# Acoustic echo cancellation. Required everywhere: on Linux there is no other
-# cancellation in the picture, and on the platforms that have their own it is
-# still the fallback for the endpoints and drivers that turn out not to.
-unix:!macx: CONFIG += link_pkgconfig
-unix:!macx: PKGCONFIG += speexdsp
+# Acoustic echo cancellation - WebRTC's AEC3, through the audio processing
+# module. Required everywhere: it is what the "use own echo cancellation"
+# setting selects, and on Linux it is usually the only cancellation in the
+# picture at all.
+#
+# Its headers, and the abseil ones they include, refuse to compile as anything
+# older than C++17, so the whole target is built that way.
+CONFIG += c++17
+
+# Through pkg-config on macOS too, unlike the libraries above: the headers sit
+# in a versioned include directory and the library needs several abseil ones
+# alongside it, so the flags are not worth writing out by hand.
+#
+# 1.x and 2.x install in parallel under separate pkg-config names and differ
+# in a few API details that echocanceller.cpp handles; 1.x wins when both are
+# there, matching what configure does. The macro is only a hint - the file
+# falls back to working the version out from the headers - but a build that
+# has both installed is exactly where the hint earns its keep.
+unix: CONFIG += link_pkgconfig
+unix {
+ packagesExist(webrtc-audio-processing-1) {
+ PKGCONFIG += webrtc-audio-processing-1
+ DEFINES += PICA_WEBRTC_AUDIO_PROCESSING_MAJOR=1
+ } else {
+ PKGCONFIG += webrtc-audio-processing-2
+ DEFINES += PICA_WEBRTC_AUDIO_PROCESSING_MAJOR=2
+ }
+}
# Listing PulseAudio sources and sinks in the settings dialog. Optional -
# playing and capturing through PulseAudio goes via FFmpeg's own "pulse"
@@ -136,4 +155,3 @@ unix:!macx: DEFINES += HAVE_LIBPULSE
# what they link against cannot.
win32: LIBS += -lole32 -loleaut32
macx: LIBS += -framework CoreAudio -framework AudioUnit -framework AudioToolbox -framework CoreFoundation
-macx: LIBS += -lspeexdsp