diff options
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 150 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.h | 65 | ||||
| -rw-r--r-- | pica-client/capturegain.cpp | 150 | ||||
| -rw-r--r-- | pica-client/capturegain.h | 12 | ||||
| -rw-r--r-- | pica-client/capturegain_coreaudio.cpp | 25 | ||||
| -rw-r--r-- | pica-client/capturegain_wasapi.cpp | 15 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 2 | ||||
| -rw-r--r-- | pica-client/echocanceller.h | 19 |
8 files changed, 368 insertions, 70 deletions
diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index 763f1b1..405746d 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -50,7 +50,9 @@ static const int kCallSampleRate = 16000; AudioVideoCallController::AudioVideoCallController(QObject *parent) : QObject(parent), callwindow(0), ringdevice(0), is_active(false), m_captureGain(nullptr), m_savedCaptureGain(0.0), m_haveSavedCaptureGain(false), - m_captureGainSteps(0), m_audioSeq(0), + m_gainCalibrating(false), m_captureGainSteps(0), m_gainStartDb(0.0), + m_haveGainDb(false), m_gainIneffectiveSteps(0), m_lastPinnedPercent(0.0), + m_gainSettleReports(0), m_audioSeq(0), m_videoSeq(0), m_videoFrameTimestamp(0), m_videoFrameStarted(false) { connect(skynet, SIGNAL(IncomingCall(QByteArray)), this, SLOT(call_from(QByteArray))); @@ -476,16 +478,71 @@ void AudioVideoCallController::capture_clipping(double pinnedPercent, double pea // Either the setting is off, or nothing on this platform would let us at // the control. The warning above is then all we can do - and it is still // worth having, since the distortion cannot be undone anywhere else. - if (!m_captureGain) + if (!m_captureGain || !m_gainCalibrating) + return; + + // Calibration is a thing that happens at the start of a call, not a gain + // rider. Once the window has passed the level is whatever it is: moving it + // mid-conversation would keep dragging the echo path out from under the + // canceller, which is worse than a bit of distortion. + if (m_callClock.elapsed() > kGainCalibrationMs) + { + closeGainCalibration("calibration window has passed"); + return; + } + + // The audio in this report may predate the last adjustment. + if (m_gainSettleReports > 0) + { + m_gainSettleReports--; return; + } - if (m_captureGainSteps >= kMaxCaptureGainSteps) + // Without a dB scale there is no safe way to know how far down we have + // come, so the step count has to be the bound - and a much tighter one, + // since six steps of 0.7 is over 50 dB on a perceptual scale. + const int stepLimit = m_haveGainDb ? kMaxCaptureGainSteps : kMaxBlindGainSteps; + if (m_captureGainSteps >= stepLimit) + { + closeGainCalibration("step limit reached"); return; + } + + // Did the previous adjustment achieve anything? Two that did not means the + // stage overloading is not the one we are holding. + if (m_captureGainSteps > 0 && pinnedPercent >= m_lastPinnedPercent) + { + m_gainIneffectiveSteps++; + if (m_gainIneffectiveSteps >= kMaxGainIneffectiveSteps) + { + closeGainCalibration("lowering this control is not reducing the clipping - " + "something ahead of it (an analog mic boost, or Windows' " + "separate Microphone Boost) is what is overloading"); + return; + } + } + else + { + m_gainIneffectiveSteps = 0; + } double current = 0.0; if (!m_captureGain->gain(¤t)) return; + // The bound that matters, and the one an earlier version of this got + // wrong: how far down we have come in decibels, not where the slider sits. + double currentDb = 0.0; + if (m_haveGainDb && m_captureGain->gainDb(¤tDb)) + { + if (m_gainStartDb - currentDb >= kMaxGainReductionDb) + { + closeGainCalibration(QString("already %1 dB down and still clipping") + .arg(m_gainStartDb - currentDb, 0, 'f', 1)); + return; + } + } + // Size the step to how badly it is clipping. A microphone left at maximum // is 20-30dB too hot and wants covering quickly; one that only tips over // on the loudest syllable wants nudging, not slamming. @@ -498,22 +555,61 @@ void AudioVideoCallController::capture_clipping(double pinnedPercent, double pea factor = 0.95; double next = current * factor; - if (next < kMinCaptureGain) - next = kMinCaptureGain; - if (next >= current) - return; // already at the floor, nothing left to give + { + closeGainCalibration("control is already at its minimum"); + return; + } if (!m_captureGain->setGain(next)) { - qWarning() << "Could not lower the capture gain - leaving it alone"; - m_captureGainSteps = kMaxCaptureGainSteps; + closeGainCalibration("the control would not move"); return; } m_captureGainSteps++; - qDebug("Capture gain %.0f%% -> %.0f%% (step %d of %d)", - current * 100.0, next * 100.0, m_captureGainSteps, kMaxCaptureGainSteps); + m_lastPinnedPercent = pinnedPercent; + m_gainSettleReports = kGainSettleReports; + + double achieved = 0.0; + double newDb = 0.0; + if (m_haveGainDb && m_captureGain->gainDb(&newDb)) + { + achieved = currentDb - newDb; + qDebug("Capture gain %.0f%% -> %.0f%% (%.1f dB, %.1f dB below the user's setting)", + current * 100.0, next * 100.0, -achieved, m_gainStartDb - newDb); + + // A step that moved the slider but not the level means we are pushing + // against the end of a control that no longer does anything. + if (achieved < kMinEffectiveStepDb) + m_gainIneffectiveSteps++; + + // Close on the step that crosses the ceiling rather than waiting for + // another report, which would let one more step through first. + if (m_gainStartDb - newDb >= kMaxGainReductionDb) + closeGainCalibration(QString("reached the %1 dB limit") + .arg(kMaxGainReductionDb, 0, 'f', 0)); + } + else + { + qDebug("Capture gain %.0f%% -> %.0f%% (step %d of %d)", + current * 100.0, next * 100.0, m_captureGainSteps, kMaxCaptureGainSteps); + } + + // The echo path just changed by however many dB that was, so everything + // the filter has learned about it is now wrong. Telling it to start over + // costs a second of convergence; not telling it costs the rest of the call. + if (m_echoCanceller) + m_echoCanceller->reset(); +} + +void AudioVideoCallController::closeGainCalibration(const QString &why) +{ + if (!m_gainCalibrating) + return; + + m_gainCalibrating = false; + qDebug() << "Microphone gain calibration finished:" << why; } void AudioVideoCallController::startCaptureGainControl(const QString &captureDevice) @@ -522,9 +618,15 @@ void AudioVideoCallController::startCaptureGainControl(const QString &captureDev m_savedCaptureGain = 0.0; m_haveSavedCaptureGain = false; m_captureGainSteps = 0; + m_gainCalibrating = false; + m_gainStartDb = 0.0; + m_haveGainDb = false; + m_gainIneffectiveSteps = 0; + m_lastPinnedPercent = 0.0; + m_gainSettleReports = 0; Settings st(config_dbname); - if (!st.loadValue("audio.auto_capture_gain", 1).toBool()) + if (!st.loadValue("audio.auto_capture_gain", 0).toBool()) return; // Whichever driver the capture side is about to open the device with - @@ -535,20 +637,29 @@ void AudioVideoCallController::startCaptureGainControl(const QString &captureDev if (!m_captureGain) return; - if (m_captureGain->gain(&m_savedCaptureGain)) - { - m_haveSavedCaptureGain = true; - qDebug() << "Automatic microphone gain using" << m_captureGain->description() - << "- currently at" << qRound(m_savedCaptureGain * 100.0) << "%"; - } - else + if (!m_captureGain->gain(&m_savedCaptureGain)) { // Without a reading there is nothing to put back afterwards, and // moving a control we cannot restore is not a trade worth making. qWarning() << "Could not read the capture gain - not taking it over"; delete m_captureGain; m_captureGain = nullptr; + return; } + + m_haveSavedCaptureGain = true; + m_haveGainDb = m_captureGain->gainDb(&m_gainStartDb); + m_gainCalibrating = true; + + if (m_haveGainDb) + qDebug("Microphone gain calibration open on %s - at %.0f%% (%.1f dB), will cut at most %.0f dB", + qPrintable(m_captureGain->description()), m_savedCaptureGain * 100.0, + m_gainStartDb, kMaxGainReductionDb); + else + qDebug("Microphone gain calibration open on %s - at %.0f%%, no dB scale so bounded " + "to %d steps", + qPrintable(m_captureGain->description()), m_savedCaptureGain * 100.0, + kMaxCaptureGainSteps); } void AudioVideoCallController::stopCaptureGainControl() @@ -572,6 +683,7 @@ void AudioVideoCallController::stopCaptureGainControl() m_captureGain = nullptr; m_haveSavedCaptureGain = false; m_captureGainSteps = 0; + m_gainCalibrating = false; } void AudioVideoCallController::incoming_audio_params(QByteArray peer_id, QString codec, quint16 sample_rate) diff --git a/pica-client/audiovideocallcontroller.h b/pica-client/audiovideocallcontroller.h index e30f4c1..f7c9e37 100644 --- a/pica-client/audiovideocallcontroller.h +++ b/pica-client/audiovideocallcontroller.h @@ -85,20 +85,64 @@ private: // // m_savedCaptureGain is where the control was found at the start of the // call, put back in stopAudioPipeline(): this is the user's own setting - // and we are only borrowing it. m_captureGainSteps bounds how far we will - // go, so a microphone that clips for some reason turning it down cannot - // fix - a broken preamp, someone shouting into it - ends up quiet rather - // than muted. + // and we are only borrowing it. CaptureGainControl *m_captureGain; double m_savedCaptureGain; bool m_haveSavedCaptureGain; - int m_captureGainSteps; - static const int kMaxCaptureGainSteps = 12; + // This is a calibration that runs at the start of a call and then stops - + // NOT a gain control that rides the level for the call's duration. The + // difference matters: an adaptive echo canceller can only track a + // stationary echo path, and moving the microphone gain moves that path. + // See the AGC comment in EchoCanceller's constructor, which says the same + // thing about the reason speexdsp's own AGC is left switched off. + // + // So every adjustment resets the canceller, and once m_gainCalibrating is + // false nothing touches the control again until the call ends. + bool m_gainCalibrating; + int m_captureGainSteps; - // Never go below this fraction of the control's travel; past it we are - // making the user inaudible to fix a problem they can hear about instead. - static constexpr double kMinCaptureGain = 0.05; + // dB reading when calibration opened, and whether the platform would give + // us one. Bounding the total cut in decibels rather than as a fraction of + // the control's travel is the whole point: a fraction means different + // things on a hardware mixer (linear in dB) and on a sound server or + // WASAPI endpoint (roughly cubic), which is how an earlier version of this + // managed to wind a microphone down to silence. + double m_gainStartDb; + bool m_haveGainDb; + + // Consecutive adjustments that moved the control but did not reduce the + // clipping. Means the stage actually overloading is one we cannot reach - + // an analog mic boost ahead of the converter, or Windows' separate + // "Microphone Boost" - and that turning our control down further will + // silence the user without ever fixing it. + int m_gainIneffectiveSteps; + double m_lastPinnedPercent; + + // Reports to ignore after an adjustment, while audio captured at the old + // setting drains out of the pipeline. + int m_gainSettleReports; + + static const int kMaxCaptureGainSteps = 6; + + // Bound used when the control will not report decibels at all, where we + // are working blind and a wrong guess is what silences a microphone. + static const int kMaxBlindGainSteps = 2; + static const int kMaxGainIneffectiveSteps = 2; + static const int kGainSettleReports = 1; + + // How long the calibration window stays open, in milliseconds of call + // time. Long enough that someone who says nothing for the first few + // seconds still gets calibrated, short enough to be over before the + // conversation proper. + static const int kGainCalibrationMs = 20000; + + // Hard ceiling on the total cut, in dB below where the user had it. Past + // this the problem is not one we are going to fix by turning a knob. + static constexpr double kMaxGainReductionDb = 24.0; + + // An adjustment that achieves less than this much is not doing anything. + static constexpr double kMinEffectiveStepDb = 0.5; // Sequence number / capture-clock timestamp stamped onto outgoing // 0x76 audio packets (see PICA_PROTO_CALL_PACKET_HDRSIZE). @@ -123,6 +167,9 @@ private: void startCaptureGainControl(const QString &captureDevice); void stopCaptureGainControl(); + // Ends the calibration window, logging why. Idempotent. + void closeGainCalibration(const QString &why); + void playEarpieceTone(void (TonePlayer::*tone)()); void playRingTone(); void stopTones(); diff --git a/pica-client/capturegain.cpp b/pica-client/capturegain.cpp index 2f2ea2d..8f89718 100644 --- a/pica-client/capturegain.cpp +++ b/pica-client/capturegain.cpp @@ -18,6 +18,7 @@ #include <QDebug> #include <QtGlobal> +#include <QStringList> #include <cmath> #ifdef Q_OS_LINUX @@ -114,48 +115,31 @@ public: bool open(const QString &pcmName) { - const QString card = alsaCardFromPcm(pcmName); - const QByteArray cardUtf8 = card.toUtf8(); - - if (snd_mixer_open(&m_mixer, 0) < 0) + // The card the PCM name points at, and then the real cards as + // fallbacks. "default" is not a card at all on a machine running a + // sound server: attaching to it lands on the server's own software + // volume, which is applied to samples the converter has already + // digitised. That control cannot undo clipping - scaling a clipped + // buffer down just gives a quieter clipped buffer - so calibrating + // against it would turn the microphone down forever without the + // clipping ever going away. Only a gain ahead of the ADC is any use + // here, and tryCard() insists on one. + QStringList candidates; + candidates << alsaCardFromPcm(pcmName); + + if (candidates.first() == QLatin1String("default")) { - m_mixer = nullptr; - return false; + for (int card = 0; card < 8; card++) + candidates << (QStringLiteral("hw:") + QString::number(card)); } - if (snd_mixer_attach(m_mixer, cardUtf8.constData()) < 0 || - snd_mixer_selem_register(m_mixer, nullptr, nullptr) < 0 || - snd_mixer_load(m_mixer) < 0) - return false; - - // "Capture" is what the recording gain is called on essentially every - // card that has one; "Mic" turns up on the ones that do not separate - // the input selector from the gain. Failing both, take the first - // element that has a capture volume at all. - m_elem = findNamed("Capture"); - if (!m_elem) - m_elem = findNamed("Mic"); - if (!m_elem) + for (int i = 0; i < candidates.size(); i++) { - for (snd_mixer_elem_t *e = snd_mixer_first_elem(m_mixer); e; e = snd_mixer_elem_next(e)) - { - if (snd_mixer_selem_is_active(e) && snd_mixer_selem_has_capture_volume(e)) - { - m_elem = e; - break; - } - } + if (tryCard(candidates[i])) + return true; } - if (!m_elem) - return false; - - if (snd_mixer_selem_get_capture_volume_range(m_elem, &m_min, &m_max) < 0 || m_max <= m_min) - return false; - - m_name = QString::fromUtf8(snd_mixer_selem_get_name(m_elem)) + - QStringLiteral(" on ") + card; - return true; + return false; } bool gain(double *out) override @@ -176,6 +160,24 @@ public: return true; } + bool gainDb(double *out) override + { + if (!m_elem || !out) + return false; + + snd_mixer_handle_events(m_mixer); + + // ALSA reports hundredths of a decibel. Not every element has a dB + // mapping - the software controls a sound server exposes often do not. + long v = 0; + if (snd_mixer_selem_get_capture_dB(m_elem, SND_MIXER_SCHN_FRONT_LEFT, &v) < 0 && + snd_mixer_selem_get_capture_dB(m_elem, SND_MIXER_SCHN_MONO, &v) < 0) + return false; + + *out = double(v) / 100.0; + return true; + } + bool setGain(double value) override { if (!m_elem) @@ -188,6 +190,60 @@ public: QString description() const override { return m_name; } private: + // A candidate has to have a decibel range as well as a volume range. That + // is what separates a converter gain from a sound server's software fader: + // the hardware control knows what its steps are worth in dB, the software + // one only has a slider position. It is also what lets the caller bound + // the total cut in dB, which is the only unit that means the same thing on + // every backend. + bool tryCard(const QString &card) + { + if (m_mixer) + { + snd_mixer_close(m_mixer); + m_mixer = nullptr; + m_elem = nullptr; + } + + if (snd_mixer_open(&m_mixer, 0) < 0) + { + m_mixer = nullptr; + return false; + } + + const QByteArray cardUtf8 = card.toUtf8(); + if (snd_mixer_attach(m_mixer, cardUtf8.constData()) < 0 || + snd_mixer_selem_register(m_mixer, nullptr, nullptr) < 0 || + snd_mixer_load(m_mixer) < 0) + return false; + + // "Capture" is what the recording gain is called on essentially every + // card that has one; "Mic" turns up on the ones that do not separate + // the input selector from the gain. + m_elem = findNamed("Capture"); + if (!m_elem) + m_elem = findNamed("Mic"); + if (!m_elem) + m_elem = findNamed("Digital"); + + if (!m_elem) + return false; + + if (snd_mixer_selem_get_capture_volume_range(m_elem, &m_min, &m_max) < 0 || m_max <= m_min) + return false; + + long dbMin = 0, dbMax = 0; + if (snd_mixer_selem_get_capture_dB_range(m_elem, &dbMin, &dbMax) < 0 || dbMax <= dbMin) + { + m_elem = nullptr; + return false; + } + + m_name = QString::fromUtf8(snd_mixer_selem_get_name(m_elem)) + + QStringLiteral(" on ") + card; + return true; + } + snd_mixer_elem_t *findNamed(const char *want) { for (snd_mixer_elem_t *e = snd_mixer_first_elem(m_mixer); e; e = snd_mixer_elem_next(e)) @@ -220,7 +276,7 @@ class PulseCaptureGain : public CaptureGainControl public: PulseCaptureGain() : m_mainloop(nullptr), m_ctx(nullptr), m_channels(1), - m_pending(false), m_ok(false), m_value(0.0) {} + m_pending(false), m_ok(false), m_value(0.0), m_valueDb(0.0) {} ~PulseCaptureGain() override { @@ -293,6 +349,24 @@ public: return m_ok; } + // pa_sw_volume_to_dB() returns -inf for a muted control, which is true but + // useless as a number to do arithmetic on. + bool gainDb(double *out) override + { + if (!out) + return false; + + double ignored; + if (!gain(&ignored)) + return false; + + if (m_valueDb < -200.0 || m_valueDb > 200.0) + return false; + + *out = m_valueDb; + return true; + } + bool setGain(double value) override { if (!m_ctx || m_source.isEmpty()) @@ -393,6 +467,7 @@ private: // The loudest channel, so a step down is a step down for all of them // rather than only for whichever happens to be quietest. self->m_value = clampGainAmplified(double(pa_cvolume_max(&info->volume)) / double(PA_VOLUME_NORM)); + self->m_valueDb = pa_sw_volume_to_dB(pa_cvolume_max(&info->volume)); self->m_ok = true; } @@ -411,6 +486,7 @@ private: bool m_pending; bool m_ok; double m_value; + double m_valueDb; }; #endif // HAVE_LIBPULSE diff --git a/pica-client/capturegain.h b/pica-client/capturegain.h index 4d833f1..a0d3bdc 100644 --- a/pica-client/capturegain.h +++ b/pica-client/capturegain.h @@ -18,6 +18,7 @@ #define CAPTUREGAIN_H #include <QString> +#include <QtGlobal> // The capture device's own gain control - the mixer slider, not a multiplier // applied to samples we already have. @@ -51,6 +52,17 @@ public: // read, in which case *out is untouched. virtual bool gain(double *out) = 0; + // The same position in decibels, where the platform can say. This is what + // a caller should bound itself by: the fraction above is a position on a + // slider whose relationship to loudness differs per backend - linear in dB + // on an ALSA hardware control, roughly cubic on PulseAudio and WASAPI - so + // "5% of travel" is a mild cut on one and near silence on another. Decibels + // mean the same thing everywhere. + // + // False when the platform will not say, and then the caller has nothing to + // go on but a step count. + virtual bool gainDb(double *out) { Q_UNUSED(out) return false; } + // Moves the control. Values outside 0.0 to 1.0 are clamped. virtual bool setGain(double value) = 0; diff --git a/pica-client/capturegain_coreaudio.cpp b/pica-client/capturegain_coreaudio.cpp index 59b1ff5..919d08d 100644 --- a/pica-client/capturegain_coreaudio.cpp +++ b/pica-client/capturegain_coreaudio.cpp @@ -183,6 +183,31 @@ public: return true; } + // kAudioDevicePropertyVolumeDecibels is the same control as the scalar + // above, reported on a decibel scale. Devices that expose the scalar do not + // always expose this one, hence the plain false rather than a fallback. + bool gainDb(double *out) override + { + if (!out || m_dev == kAudioObjectUnknown) + return false; + + AudioObjectPropertyAddress addr; + addr.mSelector = kAudioDevicePropertyVolumeDecibels; + addr.mScope = kAudioObjectPropertyScopeInput; + addr.mElement = m_master ? 0 : (m_channels.isEmpty() ? 0 : m_channels[0]); + + if (!AudioObjectHasProperty(m_dev, &addr)) + return false; + + Float32 db = 0.0f; + UInt32 size = sizeof(db); + if (AudioObjectGetPropertyData(m_dev, &addr, 0, nullptr, &size, &db) != noErr) + return false; + + *out = db; + return true; + } + bool setGain(double value) override { if (m_dev == kAudioObjectUnknown) diff --git a/pica-client/capturegain_wasapi.cpp b/pica-client/capturegain_wasapi.cpp index 15dec57..49ea84d 100644 --- a/pica-client/capturegain_wasapi.cpp +++ b/pica-client/capturegain_wasapi.cpp @@ -119,6 +119,21 @@ public: return true; } + // WASAPI keeps a real decibel value alongside the scalar, so this needs no + // conversion of our own. + bool gainDb(double *out) override + { + if (!m_vol || !out) + return false; + + float db = 0.0f; + if (FAILED(m_vol->GetMasterVolumeLevel(&db))) + return false; + + *out = db; + return true; + } + bool setGain(double value) override { if (!m_vol) diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index e1faf8d..a951b40 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -1062,7 +1062,7 @@ void SettingsDialog::loadSettings() audioPlaybackDev->setCurrentIndex(audioPlaybackDevItem); cbEchoCancel->setChecked(st.loadValue("audio.echo_cancel", 1).toBool()); - cbAutoCaptureGain->setChecked(st.loadValue("audio.auto_capture_gain", 1).toBool()); + cbAutoCaptureGain->setChecked(st.loadValue("audio.auto_capture_gain", 0).toBool()); QString audioRingDevVal = st.loadValue("audio.ring_device", "default").toString(); int audioRingDevItem = audioRingDev->findData(audioRingDevVal); diff --git a/pica-client/echocanceller.h b/pica-client/echocanceller.h index b78a2f7..e59f8d7 100644 --- a/pica-client/echocanceller.h +++ b/pica-client/echocanceller.h @@ -51,10 +51,21 @@ class EchoCanceller { public: // frameMs is the block the filter works on and must divide the audio - // handed to processNearEnd(). tailMs is how much echo delay the filter - // can model - it has to cover the sound card's output buffer plus the - // room, and costs CPU and convergence time in proportion. - EchoCanceller(int sampleRate, int frameMs = 10, int tailMs = 150); + // handed to processNearEnd(). + // + // tailMs is the longest echo delay the filter can model at all; anything + // arriving later is invisible to it and can only be dealt with + // statistically by the residual suppressor. The fixed part of that delay + // is small - the capture path plus a metre or two of flight, call it 30ms + // - so nearly all of the tail is budget for the room's reverberation. + // + // It is not free in either direction. Too short and late reflections never + // get modelled and are heard as residual echo; too long and both the + // convergence time and the steady state misadjustment grow with the tap + // count. 300ms at 16kHz is 4800 taps, which is still fewer than the 7200 + // this ran at when calls were 48kHz with a 150ms tail - so the room + // coverage doubles and the filter is still shorter than it used to be. + EchoCanceller(int sampleRate, int frameMs = 10, int tailMs = 300); ~EchoCanceller(); bool isValid() const { return m_echo != nullptr; } |
