1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
/*
(c) Copyright 2012 - 2026 Anton Sviridenko
https://picapica.im
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#ifndef ECHOCANCELLER_H
#define ECHOCANCELLER_H
#include <QString>
#include <QMutex>
#include <QVector>
#include <QSharedPointer>
struct SpeexEchoState_;
struct SpeexPreprocessState_;
// Acoustic echo cancellation for a call, using speexdsp's frequency domain
// adaptive filter (MDF).
//
// One instance is shared by the two AudioDevice objects of a call - the
// playback one feeds it the far end signal it is about to hand to the sound
// card via pushFarEnd(), the capture one runs the microphone signal through
// processNearEnd() before encoding. Both run on their own thread, so every
// entry point here takes a lock.
//
// Because AudioVideoCallController tears a call down without joining those
// two threads (see stopAudioPipeline()), ownership is shared: hold this
// through a QSharedPointer so the object outlives whichever thread leaves
// last.
//
// Alignment between the two signals is left to the FIFO: in the steady state
// the playback side pushes samples at the same rate the capture side pulls
// them, and the backlog that builds up in the FIFO settles at exactly the
// sound card's buffer depth - which is also how far ahead of the speaker the
// playback side is running. What remains is the acoustic flight time from
// speaker to microphone plus any capture side buffering, and that is what the
// filter tail is for.
class EchoCanceller
{
public:
// frameMs is the block the filter works on and must divide the audio
// handed to processNearEnd(). tailMs is how much echo delay the filter
// can model - it has to cover the sound card's output buffer plus the
// room, and costs CPU and convergence time in proportion.
EchoCanceller(int sampleRate, int frameMs = 10, int tailMs = 150);
~EchoCanceller();
bool isValid() const { return m_echo != nullptr; }
int sampleRate() const { return m_sampleRate; }
int frameSize() const { return m_frameSize; }
// Forget the adapted filter and drop any buffered far end audio. Called
// when a call starts, since the previous call's room response is not
// worth keeping and a stale FIFO would start it off misaligned.
void reset();
// Playback side: the audio about to be written to the output device.
void pushFarEnd(const qint16 *pcm, int nsamples);
// Capture side: removes the echo of what pushFarEnd() was given, in
// place. nsamples should be a whole number of frameSize() blocks; a
// partial trailing block is passed through untouched.
void processNearEnd(qint16 *pcm, int nsamples);
// Rough measure of how well it is working, in dB - how much quieter the
// output is than the microphone input while the far end is active. Zero
// until enough far end audio has gone through to mean anything.
//
// Only meaningful while the far end is actually talking, which is what
// farEndLevelDb() is for: with nothing coming out of the speaker there is
// no echo to remove and the figure means nothing.
double erle();
// Recent signal levels in dBFS, averaged over the same window as erle().
// nearEndLevelDb() is the microphone before cancellation - a floor-level
// reading there means the capture path is dead and nothing else in the
// numbers is worth reading.
double nearEndLevelDb();
double farEndLevelDb();
// Far end samples waiting to be paired with microphone audio. In the
// steady state this settles at the depth of the sound card's output
// buffer and stays there, because both are drained by the same clock.
// Climbing towards m_farCapacity means capture and playback are running
// off different clocks - separate cards - and the reference is drifting
// out of alignment with the echo it is supposed to explain.
int farEndDepth();
private:
Q_DISABLE_COPY(EchoCanceller)
int m_sampleRate;
int m_frameSize;
int m_filterLength;
// speexdsp state. The echo canceller proper, plus a preprocessor whose
// only job here is the residual echo suppression that the adaptive
// filter alone cannot do (SPEEX_PREPROCESS_SET_ECHO_STATE).
SpeexEchoState_ *m_echo;
SpeexPreprocessState_ *m_preprocess;
// Guards the speex state, which both directions reach into: the capture
// thread through speex_echo_cancellation(), and reset() from whichever
// thread starts the call.
QMutex m_procMutex;
// Far end FIFO, written by the playback thread and drained by the
// capture thread.
QMutex m_farMutex;
QVector<qint16> m_farFifo;
int m_farHead;
// Upper bound on the FIFO, in samples. Reached only if the two sound
// cards' clocks drift apart or playback outruns capture; past this the
// oldest audio is dropped, since keeping it would mean the reference
// signal lagging further and further behind the echo it is meant to
// explain.
int m_farCapacity;
// Exponentially averaged mean square of the near end (microphone), the
// cancelled output and the far end reference, for erle() and the level
// readings. Per sample rather than per call, so they stay comparable
// whatever block size the capture path hands us.
double m_nearEnergy;
double m_outEnergy;
double m_farEnergy;
QMutex m_statMutex;
// Drives the periodic qDebug() in processNearEnd(). Counted in samples
// rather than wall clock so the report interval does not depend on how
// punctually the capture thread is scheduled. Guarded by m_statMutex.
qint64 m_processedSamples;
qint64 m_nextReportSamples;
// How often the canceller reports how it is doing, in seconds.
static const int kReportIntervalSec = 2;
// Scratch far end block, reused by processNearEnd().
QVector<qint16> m_farBlock;
bool popFarEnd(qint16 *dst, int nsamples);
};
typedef QSharedPointer<EchoCanceller> EchoCancellerPtr;
#endif // ECHOCANCELLER_H
|
