/* (c) Copyright 2012 - 2018 Anton Sviridenko https://picapica.im This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ #include "audiovideocallcontroller.h" #include "globals.h" #include "skynet.h" #include "settings.h" #include #include #include // Capture/encode resolution announced to the peer in the 0x75 message. Fixed // for now - a resolution setting is a planned follow-up, see VideoDevice for // the rest of the encoding parameters. static const quint16 kVideoWidth = 640; static const quint16 kVideoHeight = 480; // Rate we capture and encode at, announced to the peer in the 0x74 message. // // 16kHz rather than 48kHz because of the echo canceller. speexdsp's MDF is // tuned for 8/16kHz, and its cost is counted in taps: a 150ms tail is 7200 // taps at 48kHz but only 2400 at 16kHz. Convergence time and steady state // misadjustment both scale with that length, so the 48kHz filter spent every // call chasing and never settling - visible in the AEC log as ERLE climbing // towards 20dB and being knocked back to zero by each doubletalk burst. // // Costs nothing worth having for speech: Opus codes this as wideband, an 8kHz // audio bandwidth, which is well past telephone quality and past where voice // intelligibility stops improving. // // The playback direction is NOT this constant - it follows whatever the peer // announces (see incoming_audio_params()), so a call with an older build that // still sends 48kHz keeps working. static const int kCallSampleRate = 16000; AudioVideoCallController::AudioVideoCallController(QObject *parent) : QObject(parent), callwindow(0), ringdevice(0), is_active(false), m_captureGain(nullptr), m_savedCaptureGain(0.0), m_haveSavedCaptureGain(false), m_captureGainSteps(0), m_audioSeq(0), m_videoSeq(0), m_videoFrameTimestamp(0), m_videoFrameStarted(false) { connect(skynet, SIGNAL(IncomingCall(QByteArray)), this, SLOT(call_from(QByteArray))); connect(skynet, SIGNAL(CallFailed(QByteArray,QString)), this, SLOT(call_failed(QByteArray,QString))); connect(skynet, SIGNAL(CallAccepted(QByteArray)), this, SLOT(call_accepted(QByteArray))); connect(skynet, SIGNAL(CallRejected(QByteArray)), this, SLOT(call_rejected(QByteArray))); connect(skynet, SIGNAL(CallHungup(QByteArray)), this, SLOT(call_hungup(QByteArray))); connect(skynet, SIGNAL(IncomingAudioParams(QByteArray,QString,quint16)), this, SLOT(incoming_audio_params(QByteArray,QString,quint16))); connect(skynet, SIGNAL(IncomingAudioPacket(QByteArray,quint16,quint32,QByteArray)), this, SLOT(incoming_audio_packet(QByteArray,quint16,quint32,QByteArray))); connect(skynet, SIGNAL(IncomingVideoParams(QByteArray,QString,quint16,quint16)), this, SLOT(incoming_video_params(QByteArray,QString,quint16,quint16))); connect(skynet, SIGNAL(IncomingVideoPacket(QByteArray,quint16,quint32,QByteArray)), this, SLOT(incoming_video_packet(QByteArray,quint16,quint32,QByteArray))); connect(skynet, SIGNAL(CallMediaTransportChanged(QByteArray,bool,QString,quint32)), this, SLOT(media_transport_changed(QByteArray,bool,QString,quint32))); // Earpiece tones (ringback/busy/unreachable) and the incoming call bell // are played by two TonePlayer instances, each living on its own // QThread since TonePlayer::play() blocks the calling thread for the // duration of the tone sequence. toneplayer = new TonePlayer(); toneplayer->moveToThread(&toneplayer_thread); toneplayer_thread.start(); ringtoneplayer = new TonePlayer(); ringtoneplayer->moveToThread(&ringtoneplayer_thread); ringtoneplayer_thread.start(); // Mic capture+encode and remote-audio decode+playback - same // one-object-per-thread pattern as the tone players above, since // AudioDevice::Capture()/Play() block their thread for the call's // duration too. microphone = new AudioDevice(); microphone->moveToThread(µphone_thread); connect(microphone, SIGNAL(packetReady(QByteArray)), this, SLOT(send_audio_packet(QByteArray))); // Queued rather than direct: this crosses from the capture thread, and // warn_capture_clipping() only logs today but is where a UI notice would // go. connect(microphone, SIGNAL(captureClipping(double,double)), this, SLOT(capture_clipping(double,double)), Qt::QueuedConnection); microphone_thread.start(); output = new AudioDevice(); output->moveToThread(&output_thread); output_thread.start(); // Camera capture+encode and remote video decode - same arrangement as // the audio devices above. cam = new VideoDevice(); cam->moveToThread(&cam_thread); connect(cam, SIGNAL(captureStarted(QString,int,int)), this, SLOT(video_capture_started(QString,int,int))); connect(cam, SIGNAL(packetReady(QByteArray,bool)), this, SLOT(send_video_packet(QByteArray,bool))); cam_thread.start(); remotevideo = new VideoDevice(); remotevideo->moveToThread(&remotevideo_thread); remotevideo_thread.start(); } AudioVideoCallController::~AudioVideoCallController() { stopTones(); stopAudioPipeline(); stopVideoPipeline(); toneplayer_thread.quit(); toneplayer_thread.wait(); delete toneplayer; ringtoneplayer_thread.quit(); ringtoneplayer_thread.wait(); delete ringtoneplayer; microphone_thread.quit(); microphone_thread.wait(); delete microphone; output_thread.quit(); output_thread.wait(); delete output; cam_thread.quit(); cam_thread.wait(); delete cam; remotevideo_thread.quit(); remotevideo_thread.wait(); delete remotevideo; } void AudioVideoCallController::playEarpieceTone(void (TonePlayer::*tone)()) { Settings st(config_dbname); QString dev = st.loadValue("audio.playback_device", "default").toString(); toneplayer->stop(); QMetaObject::invokeMethod(toneplayer, "setDeviceName", Qt::QueuedConnection, Q_ARG(QString, dev)); if (tone == &TonePlayer::playCallInProgress) QMetaObject::invokeMethod(toneplayer, "playCallInProgress", Qt::QueuedConnection); else if (tone == &TonePlayer::playUnreachable) QMetaObject::invokeMethod(toneplayer, "playUnreachable", Qt::QueuedConnection); else if (tone == &TonePlayer::playBusy) QMetaObject::invokeMethod(toneplayer, "playBusy", Qt::QueuedConnection); } void AudioVideoCallController::playRingTone() { Settings st(config_dbname); QString dev = st.loadValue("audio.ring_device", "default").toString(); ringtoneplayer->stop(); QMetaObject::invokeMethod(ringtoneplayer, "setDeviceName", Qt::QueuedConnection, Q_ARG(QString, dev)); QMetaObject::invokeMethod(ringtoneplayer, "playClassicRingtone", Qt::QueuedConnection); } void AudioVideoCallController::stopTones() { toneplayer->stop(); ringtoneplayer->stop(); } void AudioVideoCallController::startAudioPipeline() { Settings st(config_dbname); QString capDev = st.loadValue("audio.capture_device", "default").toString(); m_audioSeq = 0; m_callClock.start(); // One canceller for the call, shared by both directions: the playback // side feeds it what it is about to play, the capture side subtracts the // echo of that from the microphone. Ownership is shared because // stopAudioPipeline() does not join either thread, so this has to outlive // whichever of them returns last. // // It is created even on the platforms that cancel for us, because whether // they really will is not known until the capture stream has been opened // - see AudioDevice::captureNative(), which drops it once the backend // says the platform has the job in hand. if (st.loadValue("audio.echo_cancel", 1).toBool()) m_echoCanceller = EchoCancellerPtr(new EchoCanceller(kCallSampleRate)); else m_echoCanceller.clear(); if (m_echoCanceller && !m_echoCanceller->isValid()) m_echoCanceller.clear(); // Set before the blocking loops are started, so the queued Capture()/ // Play() calls below are what publishes it to those threads. microphone->setEchoCanceller(m_echoCanceller); output->setEchoCanceller(m_echoCanceller); // Before Capture() starts, so the gain is already read and saved by the // time the first clipping report can come back. startCaptureGainControl(capDev); // Declare our outgoing codec to the peer, then start capturing and // encoding right away - the peer's decoder is only ready once it has // processed this 0x74 message, but since the call is carried over TCP, // packets sent immediately after are guaranteed to arrive after it. skynet->SendAudioParams(m_peer_id, QStringLiteral("opus"), kCallSampleRate); QMetaObject::invokeMethod(microphone, "configureCapture", Qt::QueuedConnection, Q_ARG(QString, capDev), Q_ARG(QString, QStringLiteral("opus")), Q_ARG(int, kCallSampleRate)); QMetaObject::invokeMethod(microphone, "Capture", Qt::QueuedConnection); // output is configured lazily, once the peer's own 0x74 (see // incoming_audio_params()) tells us what codec/rate it is sending. } void AudioVideoCallController::stopAudioPipeline() { // Close() only touches a QAtomicInt and wakes a QWaitCondition, both // safe to reach from any thread - it must be called directly rather // than via invokeMethod()/QueuedConnection: microphone/output's own // thread is blocked inside Capture()/Play() for the call's duration, // so its event loop cannot deliver a queued call back to itself until // that blocking call returns - which is exactly what Close() is // supposed to make happen. Routing it through the event queue would // deadlock (the thread would never quit, and the app would hang on // exit after a call). microphone->Close(); output->Close(); // Puts the user's gain back. Safe to do while the capture thread is still // winding down: the control is only ever touched from this thread, and // capture_clipping() finds m_captureGain null and does nothing if a last // queued report arrives after this. stopCaptureGainControl(); // Drop our reference. The two audio threads still hold theirs until they // come out of Capture()/Play(), and the canceller goes away with the last // of them. m_echoCanceller.clear(); } void AudioVideoCallController::startVideoPipeline() { Settings st(config_dbname); QString camDev = st.loadValue("video.capture_device", QString()).toString(); // Unlike the audio devices, which fall back to the platform's "default" // device, there is no such name for cameras - so with nothing configured // yet (the setting is only written once the settings dialog has been // visited) pick the first camera found. if (camDev.isEmpty()) { VideoDevice enumerator; QList cams = enumerator.Enumerate(CAPTURE); if (!cams.isEmpty()) camDev = cams.first().device; } // No camera at all - the call simply carries no outgoing video. Incoming // video is unaffected, it only depends on what the peer sends. if (camDev.isEmpty()) return; m_videoSeq = 0; m_videoFrameTimestamp = 0; m_videoFrameStarted = false; // Datagram sized from the start, even though the call begins on the c2c // connection where a 0x77 message could hold far more: a media connection // may come up at any moment during the call, and frames whose fragments // were cut before that would not fit into a datagram. Over TCP this only // costs a few more message headers. cam->setMaxFragmentSize(PICA_MEDIA_SAFE_PAYLOAD); bool preferCompressed = st.loadValue("video.prefer_compressed", 0).toBool(); bool vaapiEncoding = st.loadValue("video.vaapi_encoding", 0).toBool(); // The peer is told what we are sending only once the camera is open and // the format is settled - with a compressed camera stream being forwarded // as-is, the codec is the camera's choice, not ours (see // video_capture_started()). QMetaObject::invokeMethod(cam, "configureCapture", Qt::QueuedConnection, Q_ARG(QString, camDev), Q_ARG(int, kVideoWidth), Q_ARG(int, kVideoHeight), Q_ARG(bool, preferCompressed), Q_ARG(bool, vaapiEncoding)); QMetaObject::invokeMethod(cam, "Capture", Qt::QueuedConnection); // remotevideo is configured lazily, once the peer's own 0x75 (see // incoming_video_params()) tells us what it is sending. } void AudioVideoCallController::stopVideoPipeline() { // Direct calls, for the same reason stopAudioPipeline() uses them. cam->Close(); remotevideo->Close(); m_videoAssembler.reset(); } void AudioVideoCallController::call_failed(QByteArray peer_id, QString reason) { qDebug() << "Call failed: " << reason << "\n"; if (peer_id != m_peer_id) return; is_active = false; /* play unreachable */ playEarpieceTone(&TonePlayer::playUnreachable); } void AudioVideoCallController::call_accepted(QByteArray peer_id) { /* SkyNet lives in its own thread, so this signal is queued: by the time it is delivered the local side may have already hung up and closed the call window, or moved on to a different peer entirely. Either way the call it refers to is gone - do not bring the audio pipeline up for it. */ if (peer_id != m_peer_id || !callwindow) return; stopTones(); is_active = true; callwindow->call_started(); startAudioPipeline(); startVideoPipeline(); } void AudioVideoCallController::call_rejected(QByteArray peer_id) { if (peer_id != m_peer_id) return; /* play busy tone */ playEarpieceTone(&TonePlayer::playBusy); } void AudioVideoCallController::call_hungup(QByteArray peer_id) { if (peer_id != m_peer_id) return; stopAudioPipeline(); stopVideoPipeline(); /* queued from the SkyNet thread as well - the window may already be closed */ if (callwindow) callwindow->call_ended(); /* play busy tone */ playEarpieceTone(&TonePlayer::playBusy); is_active = false; } void AudioVideoCallController::start_call(QByteArray peer_id) { if (callwindow) { callwindow->show(); /* call is already in progress */ if (is_active) return; /* re-create callwindow for a new call */ callwindow->close(); } m_peer_id = peer_id; callwindow = new CallWindow(peer_id, false); connect(callwindow, SIGNAL(start_call_pressed()), this, SLOT(initiate_call())); connect(callwindow, SIGNAL(hang_call_pressed()), this, SLOT(end_call())); connect(callwindow, SIGNAL(callwindow_closed(CallWindow*)), this, SLOT(callwindow_closed(CallWindow*))); /* decoded frames arrive from remotevideo's thread, so this is a queued connection; it dies with the window, which is deleted on close */ connect(remotevideo, SIGNAL(frameReady(QImage)), callwindow, SLOT(showRemoteFrame(QImage))); #ifdef HAVE_VAAPI connect(remotevideo, SIGNAL(hwFrameReady(AVFramePtr)), callwindow, SLOT(showRemoteHwFrame(AVFramePtr))); connect(callwindow, SIGNAL(video_rendering_failed()), this, SLOT(video_rendering_failed())); #endif callwindow->show(); } void AudioVideoCallController::call_from(QByteArray peer_id) { if (callwindow) { /* HANDLE BUSY CASE!!! */ /* call is already in progress */ return; } m_peer_id = peer_id; callwindow = new CallWindow(peer_id, true); connect(callwindow, SIGNAL(accept_call_pressed()), this, SLOT(accept_call())); connect(callwindow, SIGNAL(hang_call_pressed()), this, SLOT(end_call())); connect(callwindow, SIGNAL(callwindow_closed(CallWindow*)), this, SLOT(callwindow_closed(CallWindow*))); /* see the same connections in start_call() */ connect(remotevideo, SIGNAL(frameReady(QImage)), callwindow, SLOT(showRemoteFrame(QImage))); #ifdef HAVE_VAAPI connect(remotevideo, SIGNAL(hwFrameReady(AVFramePtr)), callwindow, SLOT(showRemoteHwFrame(AVFramePtr))); connect(callwindow, SIGNAL(video_rendering_failed()), this, SLOT(video_rendering_failed())); #endif callwindow->show(); /* play ring tone */ playRingTone(); } void AudioVideoCallController::initiate_call() { skynet->StartCall(m_peer_id); /* play ringing tone */ playEarpieceTone(&TonePlayer::playCallInProgress); } void AudioVideoCallController::accept_call() { stopTones(); skynet->AcceptCall(m_peer_id); is_active = true; callwindow->call_started(); startAudioPipeline(); startVideoPipeline(); } void AudioVideoCallController::end_call() { stopTones(); stopAudioPipeline(); stopVideoPipeline(); if (is_active) skynet->HangupCall(m_peer_id); else skynet->RejectCall(m_peer_id); is_active = false; callwindow->close(); } void AudioVideoCallController::callwindow_closed(CallWindow *sender_window) { callwindow = 0; is_active = false; } void AudioVideoCallController::send_audio_packet(QByteArray data) { if (!is_active) return; quint16 seq = m_audioSeq++; quint32 timestamp = (quint32)m_callClock.elapsed(); skynet->SendAudioPacket(m_peer_id, seq, timestamp, data); } // Rate limited by AudioDevice, which reports at most once every couple of // seconds of audio, so this can log and act unconditionally. void AudioVideoCallController::capture_clipping(double pinnedPercent, double peakDb) { qWarning("Microphone is clipping: %.2f%% of samples at full scale, peak %.1f dBFS", pinnedPercent, peakDb); // Either the setting is off, or nothing on this platform would let us at // the control. The warning above is then all we can do - and it is still // worth having, since the distortion cannot be undone anywhere else. if (!m_captureGain) return; if (m_captureGainSteps >= kMaxCaptureGainSteps) return; double current = 0.0; if (!m_captureGain->gain(¤t)) return; // Size the step to how badly it is clipping. A microphone left at maximum // is 20-30dB too hot and wants covering quickly; one that only tips over // on the loudest syllable wants nudging, not slamming. double factor; if (pinnedPercent >= 5.0) factor = 0.70; else if (pinnedPercent >= 0.5) factor = 0.85; else factor = 0.95; double next = current * factor; if (next < kMinCaptureGain) next = kMinCaptureGain; if (next >= current) return; // already at the floor, nothing left to give if (!m_captureGain->setGain(next)) { qWarning() << "Could not lower the capture gain - leaving it alone"; m_captureGainSteps = kMaxCaptureGainSteps; return; } m_captureGainSteps++; qDebug("Capture gain %.0f%% -> %.0f%% (step %d of %d)", current * 100.0, next * 100.0, m_captureGainSteps, kMaxCaptureGainSteps); } void AudioVideoCallController::startCaptureGainControl(const QString &captureDevice) { m_captureGain = nullptr; m_savedCaptureGain = 0.0; m_haveSavedCaptureGain = false; m_captureGainSteps = 0; Settings st(config_dbname); if (!st.loadValue("audio.auto_capture_gain", 1).toBool()) return; // Whichever driver the capture side is about to open the device with - // the mixer control has to be the one belonging to that device. const QString driver = AudioDevice::PlatformDriverName(CAPTURE); m_captureGain = CaptureGainControl::create(driver, captureDevice); if (!m_captureGain) return; if (m_captureGain->gain(&m_savedCaptureGain)) { m_haveSavedCaptureGain = true; qDebug() << "Automatic microphone gain using" << m_captureGain->description() << "- currently at" << qRound(m_savedCaptureGain * 100.0) << "%"; } else { // Without a reading there is nothing to put back afterwards, and // moving a control we cannot restore is not a trade worth making. qWarning() << "Could not read the capture gain - not taking it over"; delete m_captureGain; m_captureGain = nullptr; } } void AudioVideoCallController::stopCaptureGainControl() { if (!m_captureGain) return; // Put the user's own setting back. Only bother if we actually moved it, // so a call that never clipped cannot disturb a gain the user changed // themselves while it was running. if (m_haveSavedCaptureGain && m_captureGainSteps > 0) { if (m_captureGain->setGain(m_savedCaptureGain)) qDebug() << "Capture gain restored to" << qRound(m_savedCaptureGain * 100.0) << "%"; else qWarning() << "Could not restore the capture gain to" << qRound(m_savedCaptureGain * 100.0) << "% - it has been left turned down"; } delete m_captureGain; m_captureGain = nullptr; m_haveSavedCaptureGain = false; m_captureGainSteps = 0; } void AudioVideoCallController::incoming_audio_params(QByteArray peer_id, QString codec, quint16 sample_rate) { if (!is_active || peer_id != m_peer_id) return; Settings st(config_dbname); QString playDev = st.loadValue("audio.playback_device", "default").toString(); QMetaObject::invokeMethod(output, "configurePlayback", Qt::QueuedConnection, Q_ARG(QString, playDev), Q_ARG(QString, codec), Q_ARG(int, (int)sample_rate)); QMetaObject::invokeMethod(output, "Play", Qt::QueuedConnection); } void AudioVideoCallController::incoming_audio_packet(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data) { Q_UNUSED(timestamp); if (!is_active || peer_id != m_peer_id) return; // Direct call, not invokeMethod()/QueuedConnection - see the comment in // stopAudioPipeline(). enqueuePacket() is mutex-protected and safe to // call from any thread; queuing it behind output's own blocking Play() // call would mean it's never delivered, and Play() would sit forever // pulling from an empty jitter buffer. output->enqueuePacket(seq_num, data); } void AudioVideoCallController::video_capture_started(QString codec, int width, int height) { if (!is_active) return; // Announced only now, because what the camera turned out to deliver // decides it: a compressed camera stream is forwarded in the camera's own // codec and at the camera's own size, rather than in ours. skynet->SendVideoParams(m_peer_id, codec, (quint16)width, (quint16)height); } void AudioVideoCallController::send_video_packet(QByteArray data, bool is_last_fragment) { if (!is_active) return; // Every fragment of one encoded frame carries the timestamp taken when // that frame's first fragment was sent, so the receiving side can tell // the fragments of one frame from those of the next. if (!m_videoFrameStarted) { m_videoFrameTimestamp = (quint32)m_callClock.elapsed(); m_videoFrameStarted = true; } quint16 seq = m_videoSeq & VideoFrameAssembler::SeqNumMask; m_videoSeq = (m_videoSeq + 1) & VideoFrameAssembler::SeqNumMask; if (is_last_fragment) { seq |= VideoFrameAssembler::LastFragmentFlag; // The next fragment emitted belongs to a new frame. m_videoFrameStarted = false; } skynet->SendVideoPacket(m_peer_id, seq, m_videoFrameTimestamp, data); } void AudioVideoCallController::media_transport_changed(QByteArray peer_id, bool direct_udp, QString ciphersuitename, quint32 max_payload) { if (peer_id != m_peer_id) return; // Fragments stay datagram sized for the whole call (see // startVideoPipeline()); this only lowers the size further, for when the // media connection turns out to carry less than was assumed. Direct call, // not invokeMethod()/QueuedConnection - see the comment in // stopAudioPipeline(): cam's thread is blocked inside Capture(). cam->setMaxFragmentSize(qMin((int)max_payload, (int)PICA_MEDIA_SAFE_PAYLOAD)); if (callwindow) callwindow->setMediaTransport(direct_udp, ciphersuitename); qDebug() << "call media transport:" << (direct_udp ? "direct UDP/DTLS" : "c2c connection") << ciphersuitename << "max payload" << max_payload; } void AudioVideoCallController::incoming_video_params(QByteArray peer_id, QString codec, quint16 width, quint16 height) { if (!is_active || peer_id != m_peer_id) return; Settings st(config_dbname); bool vaapiDecoding = st.loadValue("video.vaapi_decoding", 0).toBool(); bool vaapiRendering = st.loadValue("video.vaapi_rendering", 0).toBool(); #ifdef HAVE_VAAPI // A call started after one where drawing GPU frames turned out not to work // would otherwise begin with a blank video area all over again. if (VaapiRenderWidget::renderingKnownBroken()) vaapiRendering = false; #endif QMetaObject::invokeMethod(remotevideo, "configurePlayback", Qt::QueuedConnection, Q_ARG(QString, codec), Q_ARG(int, (int)width), Q_ARG(int, (int)height), Q_ARG(bool, vaapiDecoding), Q_ARG(bool, vaapiRendering)); QMetaObject::invokeMethod(remotevideo, "Play", Qt::QueuedConnection); } void AudioVideoCallController::incoming_video_packet(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data) { if (!is_active || peer_id != m_peer_id) return; QByteArray frame = m_videoAssembler.addFragment(seq_num, timestamp, data); // Still waiting for the rest of this frame's fragments. if (frame.isEmpty()) return; // Direct call, not invokeMethod()/QueuedConnection - same reason as in // incoming_audio_packet(). remotevideo->enqueueFrame(frame); } #ifdef HAVE_VAAPI void AudioVideoCallController::video_rendering_failed() { // The window cannot draw frames that stayed in GPU memory, so stop keeping // them there: the decoder brings them down into system memory instead and // the window's label shows them. Decoding itself is unaffected and the // call carries on. // // Direct call rather than invokeMethod() - same reason as in // incoming_video_packet(): remotevideo's thread is blocked inside Play(). remotevideo->disableHardwareRendering(); } #endif