diff options
Diffstat (limited to 'pica-client/echocanceller.cpp')
| -rw-r--r-- | pica-client/echocanceller.cpp | 492 |
1 files changed, 304 insertions, 188 deletions
diff --git a/pica-client/echocanceller.cpp b/pica-client/echocanceller.cpp index 2d7acf0..ad5b122 100644 --- a/pica-client/echocanceller.cpp +++ b/pica-client/echocanceller.cpp @@ -15,15 +15,47 @@ along with this program. If not, see <https://www.gnu.org/licenses/>. */ #include "echocanceller.h" +#include "audiodevice.h" +#include "settings.h" +#include "globals.h" #include <QDebug> #include <QMutexLocker> #include <cmath> #include <cstring> +// Which major version of webrtc-audio-processing this is being built against. +// 1.x and 2.x are separate parallel-installable libraries with separate +// pkg-config names, and 2.x broke the API in a few places - the conditionals +// on this below are the whole of what differs as far as this file is +// concerned. +// +// configure defines it, and so does the qmake build. The fallback is for +// anything that builds this file without either: api/audio/audio_processing.h +// is where 2.x moved the header that 1.x only had under modules/, so its +// presence is the version. +#ifndef PICA_WEBRTC_AUDIO_PROCESSING_MAJOR +# if defined(__has_include) && __has_include(<api/audio/audio_processing.h>) +# define PICA_WEBRTC_AUDIO_PROCESSING_MAJOR 2 +# else +# define PICA_WEBRTC_AUDIO_PROCESSING_MAJOR 1 +# endif +#endif + +// Still the right include on 2.x, where it is a stub forwarding to +// api/audio/audio_processing.h - and it is installed there, so there is +// nothing to switch on here. +#include <modules/audio_processing/include/audio_processing.h> + +// Must come after the above, and not only as a matter of taste: on 2.x this +// header declares parameters as absl::Nullable/absl::Nonnull without +// including the abseil header that defines them, so on its own it does not +// compile. audio_processing.h is what pulls that in. Both versions include +// this themselves anyway; it is named here so that the scoped_refptr below is +// not silently relying on somebody else's include. +#include <api/scoped_refptr.h> + extern "C" { -#include <speex/speex_echo.h> -#include <speex/speex_preprocess.h> #include <libswresample/swresample.h> #include <libavutil/channel_layout.h> #include <libavutil/samplefmt.h> @@ -39,136 +71,217 @@ extern "C" { # define EC_HAVE_CH_LAYOUT 0 #endif -EchoCanceller::EchoCanceller(int sampleRate, int frameMs, int tailMs) - : m_sampleRate(sampleRate > 0 ? sampleRate : 48000), +// Setting values. Words rather than numbers because this is a three way +// choice that will be read by whoever is looking at a user's config, and a +// stray "2" says nothing. +static const char kEchoCancellationOwn[] = "own"; +static const char kEchoCancellationPlatform[] = "platform"; +static const char kEchoCancellationNone[] = "off"; + +EchoCancellationMode loadEchoCancellationSetting() +{ + Settings st(config_dbname); + + const QString value = st.loadValue("audio.echo_cancellation", QString()).toString(); + + if (value == QLatin1String(kEchoCancellationOwn)) + return EchoCancellationOwn; + + if (value == QLatin1String(kEchoCancellationPlatform)) + { + // The stored choice can stop being available underneath the user - + // the audio system moved from PulseAudio to ALSA, say. Leaving it as + // it stands would mean nobody cancelling and nothing saying so. + if (AudioDevice::PlatformEchoCancellationAvailable()) + return EchoCancellationPlatform; + + qWarning() << "Echo cancellation is set to the platform's own, but the audio system in " + "use does not provide one - falling back to ours"; + return EchoCancellationOwn; + } + + if (value == QLatin1String(kEchoCancellationNone)) + return EchoCancellationNone; + + // Nothing stored under the current key. An install that predates it may + // still have the on/off setting this replaced, and someone who turned + // cancellation off did mean it. + if (!st.loadValue("audio.echo_cancel", 1).toBool()) + return EchoCancellationNone; + + // The platform's own is the better one where it exists, so it is the + // default there; elsewhere the default is ours, which is what the plain + // on/off setting used to mean. + return AudioDevice::PlatformEchoCancellationAvailable() + ? EchoCancellationPlatform + : EchoCancellationOwn; +} + +void storeEchoCancellationSetting(EchoCancellationMode mode) +{ + Settings st(config_dbname); + + switch (mode) + { + case EchoCancellationPlatform: + st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationPlatform)); + break; + + case EchoCancellationNone: + st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationNone)); + break; + + case EchoCancellationOwn: + default: + st.storeValue("audio.echo_cancellation", QLatin1String(kEchoCancellationOwn)); + break; + } +} + +// Holds everything that needs the WebRTC headers, so that echocanceller.h - +// which audiodevice.h includes, and so half the client with it - does not. +struct EchoCancellerPrivate +{ + rtc::scoped_refptr<webrtc::AudioProcessing> apm; + + // Both directions are mono at our rate. The integer interface requires + // the capture, playback and render rates to be the same, which is why + // pushFarEnd() resamples rather than passing the peer's rate through. + webrtc::StreamConfig stream; +}; + +bool EchoCanceller::isNativeRate(int rate) +{ + return rate == 8000 || rate == 16000 || rate == 32000 || rate == 48000; +} + +EchoCanceller::EchoCanceller(int sampleRate) + : m_sampleRate(sampleRate), m_frameSize(0), - m_filterLength(0), - m_echo(nullptr), - m_preprocess(nullptr), - m_farHead(0), + m_d(nullptr), m_farSwr(nullptr), m_farSwrRate(0), - m_farCapacity(0), m_nearEnergy(0.0), - m_outEnergy(0.0), m_farEnergy(0.0), m_processedSamples(0), - m_nextReportSamples(0) + m_nextReportSamples(0), + m_renderBlocks(0), + m_captureBlocks(0) { - if (frameMs <= 0) - frameMs = 10; - if (tailMs < frameMs) - tailMs = frameMs; - - m_frameSize = (m_sampleRate * frameMs) / 1000; - m_filterLength = (m_sampleRate * tailMs) / 1000; - - // speexdsp's MDF works in blocks of frame_size and wants the filter to - // be a whole number of them. - if (m_frameSize <= 0) - m_frameSize = 480; - m_filterLength = ((m_filterLength + m_frameSize - 1) / m_frameSize) * m_frameSize; - - // Room for the sound card's output buffer plus a healthy margin. See the - // comment on m_farCapacity in the header for what overflowing it means. - m_farCapacity = m_sampleRate; // one second - - SpeexEchoState *echo = speex_echo_state_init(m_frameSize, m_filterLength); - if (!echo) + if (!isNativeRate(m_sampleRate)) { - qWarning() << "Could not create speex echo canceller state"; + qWarning() << "Echo canceller:" << m_sampleRate + << "Hz is not one of AEC3's rates (8/16/32/48 kHz) - not cancelling"; return; } - int rate = m_sampleRate; - speex_echo_ctl(echo, SPEEX_ECHO_SET_SAMPLING_RATE, &rate); + // AEC3 works on 10ms of audio at a time, everywhere, at every rate. + m_frameSize = m_sampleRate / 100; - SpeexPreprocessState *pre = speex_preprocess_state_init(m_frameSize, m_sampleRate); - if (pre) - { - // The adaptive filter removes the part of the echo it can model; what - // is left over - non-linearities in a cheap speaker, the tail of the - // room past m_filterLength - has to be suppressed statistically, and - // that is what the preprocessor's echo suppressor does with the - // residual estimate the echo state hands it. - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_STATE, echo); - - int suppress = -40; - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS, &suppress); - int suppressActive = -15; - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS_ACTIVE, &suppressActive); - - // Leave denoise on - it costs little next to the filter and keeps the - // residual suppressor's noise estimate honest. AGC stays off: the - // call has no level control anywhere else, and having one end quietly - // ride the gain makes the echo path non-stationary, which is the one - // thing an adaptive filter cannot follow. - int on = 1, off = 0; - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_DENOISE, &on); - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_AGC, &off); - speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_VAD, &off); - } - else + webrtc::AudioProcessing::Config cfg; + + cfg.echo_canceller.enabled = true; + // AEC3 proper. The mobile mode is the cut down fixed point canceller for + // phones, and is worse anywhere that can afford not to use it. + cfg.echo_canceller.mobile_mode = false; + cfg.echo_canceller.enforce_high_pass_filtering = true; + + cfg.high_pass_filter.enabled = true; + + // Costs little next to the filter, and a lower noise floor is what lets + // the residual echo suppressor tell echo from room noise. + cfg.noise_suppression.enabled = true; + cfg.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + + // Both AGCs stay off. The call has no level control anywhere else, and a + // gain riding the microphone moves the echo path, which is the one thing + // an adaptive filter cannot follow. + cfg.gain_controller1.enabled = false; + cfg.gain_controller2.enabled = false; + + cfg.transient_suppression.enabled = false; + +#if PICA_WEBRTC_AUDIO_PROCESSING_MAJOR < 2 + // All three were dropped in 2.x. The first two only gate statistics we + // never read; the third is a second, statistical echo estimator whose + // output we do not use either. Switching them off on 1.x is therefore + // the same configuration 2.x gives us for nothing. + cfg.voice_detection.enabled = false; + cfg.level_estimation.enabled = false; + cfg.residual_echo_detector.enabled = false; +#endif + + m_d = new EchoCancellerPrivate; + m_d->stream = webrtc::StreamConfig(m_sampleRate, 1); + +#if PICA_WEBRTC_AUDIO_PROCESSING_MAJOR >= 2 + m_d->apm = webrtc::AudioProcessingBuilder().Create(); +#else + // 1.x hands back a raw pointer to an object whose reference count is + // still zero - the scoped_refptr is what takes it to one, and dropping + // the result on the floor would leak rather than free. + m_d->apm = rtc::scoped_refptr<webrtc::AudioProcessing>(webrtc::AudioProcessingBuilder().Create()); +#endif + + if (!m_d->apm) { - qWarning() << "Could not create speex preprocessor - running without residual echo suppression"; + qWarning() << "Could not create the WebRTC audio processing module"; + delete m_d; + m_d = nullptr; + return; } - m_echo = reinterpret_cast<SpeexEchoState_ *>(echo); - m_preprocess = reinterpret_cast<SpeexPreprocessState_ *>(pre); + m_d->apm->ApplyConfig(cfg); - m_farBlock.resize(m_frameSize); - m_farFifo.reserve(m_farCapacity); + // Mono at our rate in both directions, which the integer interface + // requires anyway. Spelled out through a ProcessingConfig rather than the + // shorter Initialize() overload taking channel layouts, because that + // overload and the ChannelLayout enum it needs are gone in 2.x. + webrtc::ProcessingConfig streams; + streams.input_stream() = m_d->stream; + streams.output_stream() = m_d->stream; + streams.reverse_input_stream() = m_d->stream; + streams.reverse_output_stream() = m_d->stream; + + const int err = m_d->apm->Initialize(streams); + if (err != webrtc::AudioProcessing::kNoError) + { + qWarning() << "Could not initialise the WebRTC audio processing module:" << err; + delete m_d; + m_d = nullptr; + return; + } // Hold off the first report until there is a window's worth of audio // behind the averages, otherwise it only says the call just started. m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec; - qDebug() << "Echo canceller:" << m_sampleRate << "Hz, frame" << m_frameSize - << "samples, tail" << m_filterLength << "samples (" - << (m_filterLength * 1000 / m_sampleRate) << "ms )"; + // The library version is in there because it is otherwise invisible which + // of the two parallel-installable ones got linked, and they behave + // differently enough to be worth knowing about from a log alone. + qDebug("Echo canceller: WebRTC AEC3 (webrtc-audio-processing-%d), %d Hz, frame %d samples", + PICA_WEBRTC_AUDIO_PROCESSING_MAJOR, m_sampleRate, m_frameSize); } EchoCanceller::~EchoCanceller() { - // Order matters: the preprocessor holds a pointer to the echo state. - if (m_preprocess) - speex_preprocess_state_destroy(reinterpret_cast<SpeexPreprocessState *>(m_preprocess)); - if (m_echo) - speex_echo_state_destroy(reinterpret_cast<SpeexEchoState *>(m_echo)); + delete m_d; + if (m_farSwr) swr_free(&m_farSwr); } -void EchoCanceller::reset() +bool EchoCanceller::isValid() const { - { - QMutexLocker locker(&m_procMutex); - if (m_echo) - speex_echo_state_reset(reinterpret_cast<SpeexEchoState *>(m_echo)); - } - - { - QMutexLocker locker(&m_farMutex); - m_farFifo.clear(); - m_farHead = 0; - } - - { - QMutexLocker locker(&m_statMutex); - m_nearEnergy = 0.0; - m_outEnergy = 0.0; - m_farEnergy = 0.0; - m_processedSamples = 0; - m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec; - } + return m_d != nullptr; } void EchoCanceller::pushFarEnd(const qint16 *pcm, int nsamples, int srcRate) { - if (!m_echo || !pcm || nsamples <= 0) + if (!m_d || !pcm || nsamples <= 0) return; - QMutexLocker locker(&m_farMutex); + QMutexLocker locker(&m_renderMutex); if (srcRate > 0 && srcRate != m_sampleRate) { @@ -226,97 +339,59 @@ void EchoCanceller::pushFarEnd(const qint16 *pcm, int nsamples, int srcRate) nsamples = got; } - // Reclaim the space already consumed rather than growing without bound; - // doing it only once the head has run past half the buffer keeps this - // from turning into a memmove on every call. - if (m_farHead > 0 && m_farHead >= m_farFifo.size() / 2) - { - m_farFifo.remove(0, m_farHead); - m_farHead = 0; - } + int base = m_farPending.size(); + m_farPending.resize(base + nsamples); + memcpy(m_farPending.data() + base, pcm, nsamples * sizeof(qint16)); - int oldSize = m_farFifo.size(); - m_farFifo.resize(oldSize + nsamples); - memcpy(m_farFifo.data() + oldSize, pcm, nsamples * sizeof(qint16)); + double farEnergy = 0.0; + int blocks = m_farPending.size() / m_frameSize; - int pending = m_farFifo.size() - m_farHead; - if (pending > m_farCapacity) + for (int b = 0; b < blocks; b++) { - m_farHead += pending - m_farCapacity; - if (m_farHead >= m_farFifo.size()) - { - m_farFifo.clear(); - m_farHead = 0; - } - } -} + qint16 *farBlock = m_farPending.data() + b * m_frameSize; -bool EchoCanceller::popFarEnd(qint16 *dst, int nsamples) -{ - QMutexLocker locker(&m_farMutex); - - int pending = m_farFifo.size() - m_farHead; - if (pending <= 0) - return false; + for (int i = 0; i < m_frameSize; i++) + farEnergy += double(farBlock[i]) * double(farBlock[i]); - int n = pending < nsamples ? pending : nsamples; - memcpy(dst, m_farFifo.constData() + m_farHead, n * sizeof(qint16)); - m_farHead += n; + // In place: the module is happy for source and destination to be the + // same buffer, and nothing downstream wants the processed render + // signal - handing it over is only how AEC3 learns what is about to + // come back through the microphone. + m_d->apm->ProcessReverseStream(farBlock, m_d->stream, m_d->stream, farBlock); + } - if (n < nsamples) - memset(dst + n, 0, (nsamples - n) * sizeof(qint16)); + if (blocks > 0) + { + m_farPending.remove(0, blocks * m_frameSize); - return true; + QMutexLocker statLocker(&m_statMutex); + m_farEnergy = m_farEnergy * 0.95 + (farEnergy / (blocks * m_frameSize)) * 0.05; + m_renderBlocks += blocks; + } } void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) { - if (!m_echo || !pcm || nsamples < m_frameSize) + if (!m_d || !pcm || nsamples < m_frameSize) return; - QMutexLocker locker(&m_procMutex); - - SpeexEchoState *echo = reinterpret_cast<SpeexEchoState *>(m_echo); - SpeexPreprocessState *pre = reinterpret_cast<SpeexPreprocessState *>(m_preprocess); - double nearEnergy = 0.0; - double outEnergy = 0.0; - double farEnergy = 0.0; - int blocks = nsamples / m_frameSize; - for (int b = 0; b < blocks; b++) - { - // Not called "near" - windef.h still defines that as an empty macro - // on the mingw-w64 builds. - qint16 *nearBlock = pcm + b * m_frameSize; - for (int i = 0; i < m_frameSize; i++) - nearEnergy += double(nearBlock[i]) * double(nearBlock[i]); + { + QMutexLocker locker(&m_captureMutex); - if (!popFarEnd(m_farBlock.data(), m_frameSize)) + for (int b = 0; b < blocks; b++) { - // Nothing is being played, so there is no echo to remove. Still - // run the frame through, otherwise the filter's idea of the far - // end signal silently runs ahead of the microphone. - memset(m_farBlock.data(), 0, m_frameSize * sizeof(qint16)); - } - - // Measured before cancellation - this is the reference the filter is - // about to be given, so it says whether there was anything coming out - // of the speaker for it to work on. - for (int i = 0; i < m_frameSize; i++) - farEnergy += double(m_farBlock[i]) * double(m_farBlock[i]); - - // speex_echo_cancellation() is happy to work in place on the near end - // buffer, which is what lets this sit directly in the capture path - // without another copy. - speex_echo_cancellation(echo, nearBlock, m_farBlock.constData(), nearBlock); + // Not called "near" - windef.h still defines that as an empty + // macro on the mingw-w64 builds. + qint16 *nearBlock = pcm + b * m_frameSize; - if (pre) - speex_preprocess_run(pre, nearBlock); + for (int i = 0; i < m_frameSize; i++) + nearEnergy += double(nearBlock[i]) * double(nearBlock[i]); - for (int i = 0; i < m_frameSize; i++) - outEnergy += double(nearBlock[i]) * double(nearBlock[i]); + m_d->apm->ProcessStream(nearBlock, m_d->stream, m_d->stream, nearBlock); + } } // At least one whole block, guaranteed by the nsamples check on entry. @@ -326,14 +401,13 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) { QMutexLocker statLocker(&m_statMutex); - // Exponential forgetting, so erle() reflects the last few seconds + // Exponential forgetting, so the levels reflect the last few seconds // rather than the whole call. Per sample, so a change in the capture - // path's block size does not move the levels. + // path's block size does not move them. m_nearEnergy = m_nearEnergy * 0.95 + (nearEnergy / used) * 0.05; - m_outEnergy = m_outEnergy * 0.95 + (outEnergy / used) * 0.05; - m_farEnergy = m_farEnergy * 0.95 + (farEnergy / used) * 0.05; m_processedSamples += used; + m_captureBlocks += blocks; if (m_processedSamples >= m_nextReportSamples) { m_nextReportSamples = m_processedSamples + qint64(m_sampleRate) * kReportIntervalSec; @@ -347,23 +421,73 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) // does not matter for a log line. if (report) { - int depth = farEndDepth(); + int delayMs = 0; + double erleDb = 0.0; + double erlDb = 0.0; + bool haveErl = false; + + { + QMutexLocker locker(&m_captureMutex); + webrtc::AudioProcessingStats stats = m_d->apm->GetStatistics(); + + if (stats.delay_ms.has_value()) + delayMs = *stats.delay_ms; + if (stats.echo_return_loss_enhancement.has_value()) + erleDb = *stats.echo_return_loss_enhancement; + if (stats.echo_return_loss.has_value()) + { + erlDb = *stats.echo_return_loss; + haveErl = true; + } + } + + qint64 lead = 0; + { + QMutexLocker statLocker(&m_statMutex); + lead = m_renderBlocks - m_captureBlocks; + } - qDebug("AEC: erle %+.1f dB mic %.1f dBFS far %.1f dBFS ref backlog %d ms%s", - erle(), nearEndLevelDb(), farEndLevelDb(), - (depth * 1000) / m_sampleRate, - depth >= m_farCapacity ? " [FIFO FULL - reference drifting]" : ""); + // erl says how strongly the speaker couples back into the microphone, + // which is whether there is anything worth cancelling at all. AEC3 + // reports it negated (echo_remover.cc: -10*log10(erl)), so the more + // negative it is the weaker the coupling - a headset reads well down, + // speakers in a small room read close to zero. + // + // erle is how much of that coupling the filter is actually removing. + // It sits at its 0 dB floor (Erle::min = 1.f) until the filter + // converges, so a call that never leaves +0.2 dB is a call where the + // linear filter never got anywhere. + // + // The delay is what AEC3 has aligned the two streams at. Zero is not + // a delay: block_processor.cc reports 0 when the estimator has no + // value, so it means the filter is running unaligned and the erle + // beside it cannot be anything but the floor. + QString delayText = delayMs > 0 + ? QString("echo delay %1 ms").arg(delayMs) + : QStringLiteral("NOT ALIGNED"); + + QString erlText = haveErl + ? QString("erl %1 dB").arg(erlDb, 0, 'f', 1) + : QStringLiteral("erl n/a"); + + qDebug("AEC: erle %+.1f dB %s mic %.1f dBFS far %.1f dBFS %s render lead %d ms", + erleDb, qPrintable(erlText), nearEndLevelDb(), farEndLevelDb(), + qPrintable(delayText), (int)(lead * 10)); // a block is 10ms, always } } double EchoCanceller::erle() { - QMutexLocker locker(&m_statMutex); - - if (m_nearEnergy <= 0.0 || m_outEnergy <= 0.0) + if (!m_d) return 0.0; - return 10.0 * log10(m_nearEnergy / m_outEnergy); + QMutexLocker locker(&m_captureMutex); + + webrtc::AudioProcessingStats stats = m_d->apm->GetStatistics(); + + return stats.echo_return_loss_enhancement.has_value() + ? *stats.echo_return_loss_enhancement + : 0.0; } // Mean square of 16 bit samples to dBFS, with a floor so digital silence @@ -390,11 +514,3 @@ double EchoCanceller::farEndLevelDb() QMutexLocker locker(&m_statMutex); return level_db(m_farEnergy); } - -int EchoCanceller::farEndDepth() -{ - QMutexLocker locker(&m_farMutex); - - int pending = m_farFifo.size() - m_farHead; - return pending > 0 ? pending : 0; -} |
