diff options
Diffstat (limited to 'pica-client')
| -rw-r--r-- | pica-client/audiodevice.h | 19 | ||||
| -rw-r--r-- | pica-client/echocanceller.cpp | 92 | ||||
| -rw-r--r-- | pica-client/echocanceller.h | 34 |
3 files changed, 136 insertions, 9 deletions
diff --git a/pica-client/audiodevice.h b/pica-client/audiodevice.h index 988438c..ef7e5b3 100644 --- a/pica-client/audiodevice.h +++ b/pica-client/audiodevice.h @@ -188,13 +188,28 @@ private: void waitForPreroll(); bool nextPacket(QByteArray *out); - static const int kMaxQueueDepth = 8; + // Ceiling on the queue, in packets of 20ms each - one second. This is a + // runaway guard, not a latency control: Play() pops as fast as it can and + // the standing latency ends up in the sound card's buffer, not here, so + // the queue only grows when the peer sends a burst or our playback thread + // is not being scheduled. It has to be well clear of kPrerollDepth, + // because overflow is handled by dropping the OLDEST packet + // (enqueuePacket()) - that is, by throwing away audio that was about to + // be played. At the previous value of 8 that was only 160ms, close enough + // to the preroll that ordinary reordering over the direct UDP transport + // could reach it and punch holes in the speech. + static const int kMaxQueueDepth = 50; // Number of packets Play() waits to have buffered before writing the // first one, to build up a cushion against arrival jitter (packets // this call apart arrive roughly every 20ms - see // AudioVideoCallController::startAudioPipeline()). - static const int kPrerollDepth = 3; + // + // This is standing latency: whatever is prerolled here is audio the + // listener hears that much later, for the whole call. 100ms buys a + // useful margin over the 60ms this used to be while staying inside the + // range where a conversation still feels immediate. + static const int kPrerollDepth = 5; }; #endif // AUDIODEVICE_H diff --git a/pica-client/echocanceller.cpp b/pica-client/echocanceller.cpp index 60b427d..df151d6 100644 --- a/pica-client/echocanceller.cpp +++ b/pica-client/echocanceller.cpp @@ -35,7 +35,10 @@ EchoCanceller::EchoCanceller(int sampleRate, int frameMs, int tailMs) m_farHead(0), m_farCapacity(0), m_nearEnergy(0.0), - m_outEnergy(0.0) + m_outEnergy(0.0), + m_farEnergy(0.0), + m_processedSamples(0), + m_nextReportSamples(0) { if (frameMs <= 0) frameMs = 10; @@ -101,6 +104,10 @@ EchoCanceller::EchoCanceller(int sampleRate, int frameMs, int tailMs) m_farBlock.resize(m_frameSize); m_farFifo.reserve(m_farCapacity); + // Hold off the first report until there is a window's worth of audio + // behind the averages, otherwise it only says the call just started. + m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec; + qDebug() << "Echo canceller:" << m_sampleRate << "Hz, frame" << m_frameSize << "samples, tail" << m_filterLength << "samples (" << (m_filterLength * 1000 / m_sampleRate) << "ms )"; @@ -133,6 +140,9 @@ void EchoCanceller::reset() QMutexLocker locker(&m_statMutex); m_nearEnergy = 0.0; m_outEnergy = 0.0; + m_farEnergy = 0.0; + m_processedSamples = 0; + m_nextReportSamples = qint64(m_sampleRate) * kReportIntervalSec; } } @@ -198,6 +208,7 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) double nearEnergy = 0.0; double outEnergy = 0.0; + double farEnergy = 0.0; int blocks = nsamples / m_frameSize; for (int b = 0; b < blocks; b++) @@ -217,6 +228,12 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) memset(m_farBlock.data(), 0, m_frameSize * sizeof(qint16)); } + // Measured before cancellation - this is the reference the filter is + // about to be given, so it says whether there was anything coming out + // of the speaker for it to work on. + for (int i = 0; i < m_frameSize; i++) + farEnergy += double(m_farBlock[i]) * double(m_farBlock[i]); + // speex_echo_cancellation() is happy to work in place on the near end // buffer, which is what lets this sit directly in the capture path // without another copy. @@ -229,11 +246,41 @@ void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) outEnergy += double(nearBlock[i]) * double(nearBlock[i]); } - QMutexLocker statLocker(&m_statMutex); - // Exponential forgetting, so erle() reflects the last few seconds rather - // than the whole call. - m_nearEnergy = m_nearEnergy * 0.95 + nearEnergy * 0.05; - m_outEnergy = m_outEnergy * 0.95 + outEnergy * 0.05; + // At least one whole block, guaranteed by the nsamples check on entry. + const int used = blocks * m_frameSize; + bool report = false; + + { + QMutexLocker statLocker(&m_statMutex); + + // Exponential forgetting, so erle() reflects the last few seconds + // rather than the whole call. Per sample, so a change in the capture + // path's block size does not move the levels. + m_nearEnergy = m_nearEnergy * 0.95 + (nearEnergy / used) * 0.05; + m_outEnergy = m_outEnergy * 0.95 + (outEnergy / used) * 0.05; + m_farEnergy = m_farEnergy * 0.95 + (farEnergy / used) * 0.05; + + m_processedSamples += used; + if (m_processedSamples >= m_nextReportSamples) + { + m_nextReportSamples = m_processedSamples + qint64(m_sampleRate) * kReportIntervalSec; + report = true; + } + } + + // Outside the lock: the accessors below take it themselves, and qDebug() + // does I/O that has no business happening with a mutex held on the + // capture thread. The values can shift by one block in between, which + // does not matter for a log line. + if (report) + { + int depth = farEndDepth(); + + qDebug("AEC: erle %+.1f dB mic %.1f dBFS far %.1f dBFS ref backlog %d ms%s", + erle(), nearEndLevelDb(), farEndLevelDb(), + (depth * 1000) / m_sampleRate, + depth >= m_farCapacity ? " [FIFO FULL - reference drifting]" : ""); + } } double EchoCanceller::erle() @@ -245,3 +292,36 @@ double EchoCanceller::erle() return 10.0 * log10(m_nearEnergy / m_outEnergy); } + +// Mean square of 16 bit samples to dBFS, with a floor so digital silence +// prints as a number rather than -inf. +static double level_db(double meanSquare) +{ + static const double kFullScale = 32768.0 * 32768.0; + + if (meanSquare <= 0.0) + return -120.0; + + double db = 10.0 * log10(meanSquare / kFullScale); + return db < -120.0 ? -120.0 : db; +} + +double EchoCanceller::nearEndLevelDb() +{ + QMutexLocker locker(&m_statMutex); + return level_db(m_nearEnergy); +} + +double EchoCanceller::farEndLevelDb() +{ + QMutexLocker locker(&m_statMutex); + return level_db(m_farEnergy); +} + +int EchoCanceller::farEndDepth() +{ + QMutexLocker locker(&m_farMutex); + + int pending = m_farFifo.size() - m_farHead; + return pending > 0 ? pending : 0; +} diff --git a/pica-client/echocanceller.h b/pica-client/echocanceller.h index 476e88e..c49962d 100644 --- a/pica-client/echocanceller.h +++ b/pica-client/echocanceller.h @@ -76,8 +76,27 @@ public: // Rough measure of how well it is working, in dB - how much quieter the // output is than the microphone input while the far end is active. Zero // until enough far end audio has gone through to mean anything. + // + // Only meaningful while the far end is actually talking, which is what + // farEndLevelDb() is for: with nothing coming out of the speaker there is + // no echo to remove and the figure means nothing. double erle(); + // Recent signal levels in dBFS, averaged over the same window as erle(). + // nearEndLevelDb() is the microphone before cancellation - a floor-level + // reading there means the capture path is dead and nothing else in the + // numbers is worth reading. + double nearEndLevelDb(); + double farEndLevelDb(); + + // Far end samples waiting to be paired with microphone audio. In the + // steady state this settles at the depth of the sound card's output + // buffer and stays there, because both are drained by the same clock. + // Climbing towards m_farCapacity means capture and playback are running + // off different clocks - separate cards - and the reference is drifting + // out of alignment with the echo it is supposed to explain. + int farEndDepth(); + private: Q_DISABLE_COPY(EchoCanceller) @@ -109,11 +128,24 @@ private: // explain. int m_farCapacity; - // Running sums of near end and output energy, for erle(). + // Exponentially averaged mean square of the near end (microphone), the + // cancelled output and the far end reference, for erle() and the level + // readings. Per sample rather than per call, so they stay comparable + // whatever block size the capture path hands us. double m_nearEnergy; double m_outEnergy; + double m_farEnergy; QMutex m_statMutex; + // Drives the periodic qDebug() in processNearEnd(). Counted in samples + // rather than wall clock so the report interval does not depend on how + // punctually the capture thread is scheduled. Guarded by m_statMutex. + qint64 m_processedSamples; + qint64 m_nextReportSamples; + + // How often the canceller reports how it is doing, in seconds. + static const int kReportIntervalSec = 2; + // Scratch far end block, reused by processNearEnd(). QVector<qint16> m_farBlock; |