diff options
| author | Anton <anton@picapica.im> | 2026-09-26 19:00:40 +0400 |
|---|---|---|
| committer | Anton <anton@picapica.im> | 2026-09-26 19:00:40 +0400 |
| commit | 76225a538e1e960eda9e31a34846f4bc62adef87 (patch) | |
| tree | e1a62b1c2c55b5c6c1e319d3c6726da4f4f96f7f /pica-client | |
| parent | bc708f0cc6c01999a8e111d1c66c899106e91bc7 (diff) | |
| download | pica-pica-76225a538e1e960eda9e31a34846f4bc62adef87.tar.gz pica-pica-76225a538e1e960eda9e31a34846f4bc62adef87.zip | |
pica-client: configurable resolutions, codecs, bitrates for a call
Diffstat (limited to 'pica-client')
| -rw-r--r-- | pica-client/audiodevice.cpp | 136 | ||||
| -rw-r--r-- | pica-client/audiodevice.h | 9 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 57 | ||||
| -rw-r--r-- | pica-client/callsettings.cpp | 124 | ||||
| -rw-r--r-- | pica-client/callsettings.h | 106 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 400 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.h | 39 | ||||
| -rw-r--r-- | pica-client/mediadevice.h | 4 | ||||
| -rw-r--r-- | pica-client/pica-client.pro | 2 | ||||
| -rw-r--r-- | pica-client/videodevice.cpp | 467 | ||||
| -rw-r--r-- | pica-client/videodevice.h | 68 | ||||
| -rw-r--r-- | pica-client/videodshow.cpp | 240 |
12 files changed, 1519 insertions, 133 deletions
diff --git a/pica-client/audiodevice.cpp b/pica-client/audiodevice.cpp index fdbeedc..fdfd449 100644 --- a/pica-client/audiodevice.cpp +++ b/pica-client/audiodevice.cpp @@ -15,6 +15,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>. */ #include "audiodevice.h" +#include "callsettings.h" #include "nativeaudio.h" #include <QDebug> @@ -68,9 +69,10 @@ extern "C" { // TonePlayer's design and keeps the resampling/framing code below simple. static const int kChannels = 1; -// Target bitrate for the Opus voice encoder. Not exposed as a setting yet; -// a reasonable fixed default for a single speech channel. -static const int64_t kOpusBitrate = 32000; +// Bitrate an encoder runs at when configureCapture() was given none, and the +// one the "Audio Bitrate" setting defaults to: a reasonable rate for a single +// speech channel of Opus. +static const int64_t kOpusBitrate = kDefaultAudioBitrateKbps * 1000; // The FFmpeg driver name used on Linux. Kept in a process wide variable // rather than read from the settings database on demand, because the audio @@ -140,6 +142,65 @@ static SwrContext *makeMonoResampler(AVSampleFormat inFmt, int inRate, AVSampleF return swr; } +// The AVCodecID behind one of the names the 0x74 message carries. +// +// Those are FFmpeg *codec* names (see doc/proto-doc-latest.txt), which are not +// always the name of an encoder or decoder for them: FFmpeg names each +// implementation after itself, so the G.722 codec is "adpcm_g722" while the +// only encoder and decoder for it are both called "g722". Going through the +// codec descriptor rather than through the implementation name is what makes +// the protocol's names work for all three. +static AVCodecID callAudioCodecId(const QString &codec) +{ + const AVCodecDescriptor *desc = avcodec_descriptor_get_by_name(codec.toLatin1().constData()); + + if (!desc || desc->type != AVMEDIA_TYPE_AUDIO) + return AV_CODEC_ID_NONE; + + return desc->id; +} + +// The encoder for one of the call audio codecs. Only "opus" needs a word of +// its own, because FFmpeg has two Opus encoders and libopus is both the better +// one and the only one offering the interleaved 16 bit format the echo +// canceller wants. +static const AVCodec *findCallAudioEncoder(const QString &codec) +{ + if (codec == QLatin1String("opus")) + { + const AVCodec *enc = avcodec_find_encoder_by_name("libopus"); + + if (enc) + return enc; + } + + const AVCodecID id = callAudioCodecId(codec); + + // AV_CODEC_ID_NONE covers a name that is not a codec at all, and one that + // names a video codec - which would otherwise be opened as one here and + // produce something no audio decoder on the far side can read. + return id == AV_CODEC_ID_NONE ? nullptr : avcodec_find_encoder(id); +} + +// The decoder counterpart, for whatever codec the peer announced. Resolved by +// name for the same reason VideoDevice::Play() does it: the 0x74 message names +// the codec the way FFmpeg does, so a peer that picks a different one of the +// three needs no codec list kept in step here. +static const AVCodec *findCallAudioDecoder(const QString &codec) +{ + if (codec == QLatin1String("opus")) + { + const AVCodec *dec = avcodec_find_decoder_by_name("libopus"); + + if (dec) + return dec; + } + + const AVCodecID id = callAudioCodecId(codec); + + return id == AV_CODEC_ID_NONE ? nullptr : avcodec_find_decoder(id); +} + // Which sample format to run the Opus encoder in. Interleaved 16 bit is // preferred over whatever the encoder lists first, because that is the format // the echo canceller works in - taking it means the microphone samples can go @@ -182,7 +243,7 @@ static AVSampleFormat chooseEncoderSampleFmt(const AVCodec *enc) } AudioDevice::AudioDevice(QObject *parent) - : QObject(parent), m_sampleRate(48000), m_abort(0), + : QObject(parent), m_sampleRate(48000), m_bitrate(kOpusBitrate), m_abort(0), m_clipCount(0), m_levelCount(0), m_levelPeak(0), m_lastPlayedSeq(0), m_havePlayedSeq(false) { @@ -229,11 +290,12 @@ AudioDevice::~AudioDevice() Close(); } -void AudioDevice::configureCapture(QString deviceName, QString codec, int sampleRate) +void AudioDevice::configureCapture(QString deviceName, QString codec, int sampleRate, int bitrate) { m_deviceName = deviceName; m_codec = codec; m_sampleRate = sampleRate; + m_bitrate = bitrate > 0 ? bitrate : kOpusBitrate; } void AudioDevice::configurePlayback(QString deviceName, QString codec, int sampleRate) @@ -431,15 +493,14 @@ void AudioDevice::captureFFmpeg(const QString &driver) return; } - const AVCodec *enc = avcodec_find_encoder_by_name("libopus"); - if (!enc) - enc = avcodec_find_encoder(AV_CODEC_ID_OPUS); + const AVCodec *enc = findCallAudioEncoder(m_codec); AVCodecContext *enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr; if (!enc || !enc_ctx) { - QString msg = "Opus encoder not available"; + QString msg = QString("No %1 encoder available").arg(m_codec); qWarning() << msg; emit errorOccurred(msg); + if (enc_ctx) avcodec_free_context(&enc_ctx); avcodec_free_context(&dec_ctx); avformat_close_input(&ifmt_ctx); return; @@ -447,19 +508,21 @@ void AudioDevice::captureFFmpeg(const QString &driver) enc_ctx->sample_rate = m_sampleRate; enc_ctx->sample_fmt = chooseEncoderSampleFmt(enc); - enc_ctx->bit_rate = kOpusBitrate; + enc_ctx->bit_rate = m_bitrate; enc_ctx->time_base = AVRational{1, m_sampleRate}; applyMonoLayout(enc_ctx); // "voip" favors speech intelligibility over faithfulness and has lower // algorithmic delay than the default "audio" mode - a better fit for a // live call than for e.g. music. Only libopus exposes this private - // option; harmless no-op on the native "opus" encoder fallback. - av_opt_set(enc_ctx->priv_data, "application", "voip", 0); + // option; the other codecs have no private options at all, and a PCM + // encoder has no priv_data to pass here in the first place. + if (m_codec == QLatin1String("opus") && enc_ctx->priv_data) + av_opt_set(enc_ctx->priv_data, "application", "voip", 0); if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) < 0) { - QString msg = QString("Could not open Opus encoder: %1").arg(ff_errstr(ret)); + QString msg = QString("Could not open the %1 encoder: %2").arg(m_codec, ff_errstr(ret)); qWarning() << msg; emit errorOccurred(msg); avcodec_free_context(&enc_ctx); @@ -481,7 +544,11 @@ void AudioDevice::captureFFmpeg(const QString &driver) } AVAudioFifo *fifo = av_audio_fifo_alloc(enc_ctx->sample_fmt, kChannels, 1); - int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : 960; + // An encoder that takes any number of samples per packet - a PCM one - + // leaves this at zero and the packet size is ours to pick. 20ms is what + // the other two produce and what the playback side's jitter buffer is + // sized in, so it is what everything here already assumes. + int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : m_sampleRate / 50; // The canceller only gets to work in place when the encoder is taking // interleaved 16 bit, which libopus does. On the native encoder, whose @@ -590,7 +657,7 @@ void AudioDevice::captureFFmpeg(const QString &driver) } else if (!loggedEncodeError) { - qWarning() << "Opus encode failed"; + qWarning() << QString("%1 encode failed").arg(m_codec); loggedEncodeError = true; } av_frame_free(&enc_frame); @@ -638,13 +705,11 @@ void AudioDevice::captureNative(const QString &api) emit deviceFormatInUse(backend->formatDescription()); - const AVCodec *enc = avcodec_find_encoder_by_name("libopus"); - if (!enc) - enc = avcodec_find_encoder(AV_CODEC_ID_OPUS); + const AVCodec *enc = findCallAudioEncoder(m_codec); AVCodecContext *enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr; if (!enc || !enc_ctx) { - QString msg = "Opus encoder not available"; + QString msg = QString("No %1 encoder available").arg(m_codec); qWarning() << msg; emit errorOccurred(msg); if (enc_ctx) avcodec_free_context(&enc_ctx); @@ -655,15 +720,18 @@ void AudioDevice::captureNative(const QString &api) enc_ctx->sample_rate = m_sampleRate; enc_ctx->sample_fmt = chooseEncoderSampleFmt(enc); - enc_ctx->bit_rate = kOpusBitrate; + enc_ctx->bit_rate = m_bitrate; enc_ctx->time_base = AVRational{1, m_sampleRate}; applyMonoLayout(enc_ctx); - av_opt_set(enc_ctx->priv_data, "application", "voip", 0); + + // See captureFFmpeg() for why this is guarded. + if (m_codec == QLatin1String("opus") && enc_ctx->priv_data) + av_opt_set(enc_ctx->priv_data, "application", "voip", 0); int ret = avcodec_open2(enc_ctx, enc, nullptr); if (ret < 0) { - QString msg = QString("Could not open Opus encoder: %1").arg(ff_errstr(ret)); + QString msg = QString("Could not open the %1 encoder: %2").arg(m_codec, ff_errstr(ret)); qWarning() << msg; emit errorOccurred(msg); avcodec_free_context(&enc_ctx); @@ -706,7 +774,8 @@ void AudioDevice::captureNative(const QString &api) ec.clear(); } - const int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : 960; + // Same reasoning as in captureFFmpeg(). + const int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : m_sampleRate / 50; QVector<qint16> pending; QVector<qint16> scratch(frame_size * 4); @@ -783,7 +852,7 @@ void AudioDevice::captureNative(const QString &api) } else if (!loggedEncodeError) { - qWarning() << "Opus encode failed"; + qWarning() << QString("%1 encode failed").arg(m_codec); loggedEncodeError = true; } @@ -884,16 +953,18 @@ void AudioDevice::playFFmpeg(const QString &driver) return; } - // Only "opus" is understood by this slice - matches what - // AudioVideoCallController always negotiates via SendAudioParams(). - const AVCodec *in_codec = avcodec_find_decoder(AV_CODEC_ID_OPUS); + // Whatever the peer announced in its 0x74 message, looked up by the name + // it used - the peer's choice of codec is its own and need not match ours. + const AVCodec *in_codec = findCallAudioDecoder(m_codec); AVCodecContext *dec_ctx = in_codec ? avcodec_alloc_context3(in_codec) : nullptr; if (dec_ctx) { + // A PCM decoder is told nothing by the bitstream and has to be given + // both of these; the others work them out for themselves. dec_ctx->sample_rate = m_sampleRate; applyMonoLayout(dec_ctx); } - if (m_codec != QLatin1String("opus") || !in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0) + if (!in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0) { QString msg = QString("Unsupported call audio codec: %1").arg(m_codec); qWarning() << msg; @@ -945,7 +1016,7 @@ void AudioDevice::playFFmpeg(const QString &driver) { if (!loggedDecodeError) { - qWarning() << QString("Opus decode failed: %1").arg(ff_errstr(ret)); + qWarning() << QString("%1 decode failed: %2").arg(m_codec, ff_errstr(ret)); loggedDecodeError = true; } continue; @@ -1054,14 +1125,15 @@ void AudioDevice::playNative(const QString &api) emit deviceFormatInUse(backend->formatDescription()); - const AVCodec *in_codec = avcodec_find_decoder(AV_CODEC_ID_OPUS); + // Same as in playFFmpeg(): the codec the peer announced, by its name. + const AVCodec *in_codec = findCallAudioDecoder(m_codec); AVCodecContext *dec_ctx = in_codec ? avcodec_alloc_context3(in_codec) : nullptr; if (dec_ctx) { dec_ctx->sample_rate = m_sampleRate; applyMonoLayout(dec_ctx); } - if (m_codec != QLatin1String("opus") || !in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0) + if (!in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0) { QString msg = QString("Unsupported call audio codec: %1").arg(m_codec); qWarning() << msg; @@ -1110,7 +1182,7 @@ void AudioDevice::playNative(const QString &api) { if (!loggedDecodeError) { - qWarning() << QString("Opus decode failed: %1").arg(ff_errstr(ret)); + qWarning() << QString("%1 decode failed: %2").arg(m_codec, ff_errstr(ret)); loggedDecodeError = true; } continue; diff --git a/pica-client/audiodevice.h b/pica-client/audiodevice.h index b98acf2..c1feb8a 100644 --- a/pica-client/audiodevice.h +++ b/pica-client/audiodevice.h @@ -47,7 +47,12 @@ public: // device identifier as accepted by the driver named by // PlatformDriverName() (e.g. an ALSA PCM name, a PulseAudio source name, // a WASAPI endpoint id, a CoreAudio device UID), or "default". - Q_INVOKABLE void configureCapture(QString deviceName, QString codec, int sampleRate); + // + // codec is the FFmpeg name of the codec, which is also the name the 0x74 + // message carries - "opus", "adpcm_g722" or "pcm_alaw". bitrate is in bits + // per second and only reaches an encoder with a rate to control: G.722 and + // A-law code a fixed number of bits per sample and simply ignore it. + Q_INVOKABLE void configureCapture(QString deviceName, QString codec, int sampleRate, int bitrate); Q_INVOKABLE void configurePlayback(QString deviceName, QString codec, int sampleRate); // Give this device the call's echo canceller. The capture direction runs @@ -173,6 +178,8 @@ private: QString m_deviceName; QString m_codec; int m_sampleRate; + // Capture direction only; see configureCapture(). + int m_bitrate; QAtomicInt m_abort; diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index 72b6fc2..9b4ae4d 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -18,32 +18,11 @@ #include "globals.h" #include "skynet.h" #include "settings.h" +#include "callsettings.h" #include <QByteArray> #include <QDebug> #include <QMetaObject> -// Capture/encode resolution announced to the peer in the 0x75 message. Fixed -// for now - a resolution setting is a planned follow-up, see VideoDevice for -// the rest of the encoding parameters. -static const quint16 kVideoWidth = 640; -static const quint16 kVideoHeight = 480; - -// Rate we capture and encode at, announced to the peer in the 0x74 message. -// -// 16kHz rather than 48kHz because it is what the echo canceller wants. AEC3 -// does its filtering in the 0-8kHz band whatever it is handed; at 48kHz it -// splits the signal into four bands, cancels in the lowest and gates the rest, -// which is more work for a result that is no better on speech. -// -// Costs nothing worth having for speech: Opus codes this as wideband, an 8kHz -// audio bandwidth, which is well past telephone quality and past where voice -// intelligibility stops improving. -// -// The playback direction is NOT this constant - it follows whatever the peer -// announces (see incoming_audio_params()), so a call with an older build that -// still sends 48kHz keeps working. -static const int kCallSampleRate = 16000; - AudioVideoCallController::AudioVideoCallController(QObject *parent) : QObject(parent), callwindow(0), ringdevice(0), is_active(false), @@ -174,6 +153,21 @@ void AudioVideoCallController::startAudioPipeline() { Settings st(config_dbname); QString capDev = st.loadValue("audio.capture_device", "default").toString(); + const CallSettings cs = CallSettings::load(); + + // What we capture and encode at, announced to the peer in the 0x74 + // message. It follows from the codec - see callAudioSampleRate(). + // + // The playback direction is NOT this: it follows whatever the peer + // announces in its own 0x74 (see incoming_audio_params()), so a call with + // a peer that picked a different codec or rate keeps working. + const int sampleRate = callAudioSampleRate(cs.audioCodec); + + // G.722 and A-law spend a fixed number of bits per sample and have no rate + // to ask for; the setting only means anything to Opus. + const int bitrate = callAudioBitrateAdjustable(cs.audioCodec) + ? cs.audioBitrateKbps * 1000 + : 0; m_audioSeq = 0; m_callClock.start(); @@ -188,8 +182,12 @@ void AudioVideoCallController::startAudioPipeline() // below us is doing the job or nobody is, and either way running this as // well would only take a second bite out of the speech - the devices are // opened accordingly, see AudioDevice::captureNative(). + // + // Built at the rate the codec runs at rather than at a fixed one, so that + // choosing a codec does not quietly switch cancellation off: the capture + // side drops a canceller whose rate does not match its own. if (AudioDevice::EchoCancellation() == EchoCancellationOwn) - m_echoCanceller = EchoCancellerPtr(new EchoCanceller(kCallSampleRate)); + m_echoCanceller = EchoCancellerPtr(new EchoCanceller(sampleRate)); else m_echoCanceller.clear(); @@ -205,10 +203,11 @@ void AudioVideoCallController::startAudioPipeline() // encoding right away - the peer's decoder is only ready once it has // processed this 0x74 message, but since the call is carried over TCP, // packets sent immediately after are guaranteed to arrive after it. - skynet->SendAudioParams(m_peer_id, QStringLiteral("opus"), kCallSampleRate); + skynet->SendAudioParams(m_peer_id, cs.audioCodec, (quint16)sampleRate); QMetaObject::invokeMethod(microphone, "configureCapture", Qt::QueuedConnection, - Q_ARG(QString, capDev), Q_ARG(QString, QStringLiteral("opus")), Q_ARG(int, kCallSampleRate)); + Q_ARG(QString, capDev), Q_ARG(QString, cs.audioCodec), + Q_ARG(int, sampleRate), Q_ARG(int, bitrate)); QMetaObject::invokeMethod(microphone, "Capture", Qt::QueuedConnection); // output is configured lazily, once the peer's own 0x74 (see @@ -239,6 +238,7 @@ void AudioVideoCallController::startVideoPipeline() { Settings st(config_dbname); QString camDev = st.loadValue("video.capture_device", QString()).toString(); + const CallSettings cs = CallSettings::load(); // Unlike the audio devices, which fall back to the platform's "default" // device, there is no such name for cameras - so with nothing configured @@ -278,8 +278,11 @@ void AudioVideoCallController::startVideoPipeline() // video_capture_started()). QMetaObject::invokeMethod(cam, "configureCapture", Qt::QueuedConnection, Q_ARG(QString, camDev), - Q_ARG(int, kVideoWidth), Q_ARG(int, kVideoHeight), - Q_ARG(bool, preferCompressed), Q_ARG(bool, vaapiEncoding)); + Q_ARG(int, cs.captureWidth), Q_ARG(int, cs.captureHeight), + Q_ARG(int, cs.captureFrameRate), + Q_ARG(bool, preferCompressed), + Q_ARG(QString, cs.videoCodec), Q_ARG(int, cs.videoBitrateKbps * 1000), + Q_ARG(bool, vaapiEncoding)); QMetaObject::invokeMethod(cam, "Capture", Qt::QueuedConnection); // remotevideo is configured lazily, once the peer's own 0x75 (see diff --git a/pica-client/callsettings.cpp b/pica-client/callsettings.cpp new file mode 100644 index 0000000..b97d46a --- /dev/null +++ b/pica-client/callsettings.cpp @@ -0,0 +1,124 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "callsettings.h" +#include "settings.h" +#include "globals.h" + +const CallCodecInfo kCallVideoCodecs[] = +{ + { "h264", "H.264" }, + { "hevc", "H.265/HEVC" }, + { "vp9", "VP9" } +}; + +const int kCallVideoCodecCount = sizeof(kCallVideoCodecs) / sizeof(kCallVideoCodecs[0]); + +const CallCodecInfo kCallAudioCodecs[] = +{ + { "opus", "Opus" }, + { "adpcm_g722", "G.722" }, + { "pcm_alaw", "G.711 A-law" } +}; + +const int kCallAudioCodecCount = sizeof(kCallAudioCodecs) / sizeof(kCallAudioCodecs[0]); + +int callAudioSampleRate(const QString &codec) +{ + // G.722 is a 16kHz codec by definition - it subband-codes a 7kHz audio + // bandwidth and there is no other rate to run it at. + if (codec == QLatin1String("adpcm_g722")) + return 16000; + + // A-law is the telephone network's codec and 8kHz is what it is for. + // Running it at anything else would only spend twice the bandwidth on a + // codec chosen for costing as little as possible. + if (codec == QLatin1String("pcm_alaw")) + return 8000; + + // Opus, and anything unrecognised: 16kHz rather than 48kHz because it is + // what the echo canceller wants. AEC3 does its filtering in the 0-8kHz + // band whatever it is handed; at 48kHz it splits the signal into four + // bands, cancels in the lowest and gates the rest, which is more work for + // a result that is no better on speech. + // + // Costs nothing worth having for speech: Opus codes this as wideband, an + // 8kHz audio bandwidth, which is well past telephone quality and past + // where voice intelligibility stops improving. + // + // Note that this is the capture direction only. Playback follows whatever + // the peer announced in its own 0x74 message (see + // AudioVideoCallController::incoming_audio_params()), so a call with a + // peer that picked a different codec or rate keeps working. + return 16000; +} + +bool callAudioBitrateAdjustable(const QString &codec) +{ + // Opus is the only one of the three with a rate control worth the name; + // the other two are fixed bits per sample. + return codec == QLatin1String("opus") || codec.isEmpty(); +} + +int callAudioFixedBitrateKbps(const QString &codec) +{ + // G.722 is 4 bits per sample at 16kHz, A-law 8 bits per sample at 8kHz - + // 64 kbit/s either way, which is no coincidence. + if (codec == QLatin1String("adpcm_g722")) + return 16000 * 4 / 1000; + + if (codec == QLatin1String("pcm_alaw")) + return 8000 * 8 / 1000; + + return kDefaultAudioBitrateKbps; +} + +CallSettings CallSettings::load() +{ + Settings st(config_dbname); + CallSettings cs; + + cs.videoCodec = st.loadValue("call.video_codec", QLatin1String(kCallVideoCodecs[0].name)).toString(); + cs.audioCodec = st.loadValue("call.audio_codec", QLatin1String(kCallAudioCodecs[0].name)).toString(); + + cs.videoBitrateKbps = st.loadValue("call.video_bitrate_kbps", kDefaultVideoBitrateKbps).toInt(); + cs.audioBitrateKbps = st.loadValue("call.audio_bitrate_kbps", kDefaultAudioBitrateKbps).toInt(); + + cs.captureWidth = st.loadValue("video.capture_width", kDefaultCaptureWidth).toInt(); + cs.captureHeight = st.loadValue("video.capture_height", kDefaultCaptureHeight).toInt(); + cs.captureFrameRate = st.loadValue("video.capture_framerate", kDefaultCaptureFrameRate).toInt(); + + // A stored value that makes no sense - a hand-edited database, or a + // setting written by a build that offered something this one does not - + // would otherwise be passed to the encoder and fail the call rather than + // the setting. + if (cs.videoBitrateKbps <= 0) + cs.videoBitrateKbps = kDefaultVideoBitrateKbps; + + if (cs.audioBitrateKbps <= 0) + cs.audioBitrateKbps = kDefaultAudioBitrateKbps; + + if (cs.captureWidth <= 0 || cs.captureHeight <= 0) + { + cs.captureWidth = kDefaultCaptureWidth; + cs.captureHeight = kDefaultCaptureHeight; + } + + if (cs.captureFrameRate <= 0) + cs.captureFrameRate = kDefaultCaptureFrameRate; + + return cs; +} diff --git a/pica-client/callsettings.h b/pica-client/callsettings.h new file mode 100644 index 0000000..65419e4 --- /dev/null +++ b/pica-client/callsettings.h @@ -0,0 +1,106 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#ifndef CALLSETTINGS_H +#define CALLSETTINGS_H + +#include <QString> + +// The parameters an audio/video call is started with: what the settings +// dialog's "Call Settings" and "Video Devices" tabs write, and what +// AudioVideoCallController reads when a call begins. Both sides go through +// this one place so they cannot drift apart, and so that "never configured" +// means the same thing to each of them. +// +// Codec names are FFmpeg names throughout. That is also how the 0x74 and 0x75 +// protocol messages name them (see doc/proto-doc-latest.txt), so what is +// stored here goes both onto the wire and into +// avcodec_find_encoder_by_name() with no translation table in between. + +// One entry of the codec lists the settings dialog offers. +struct CallCodecInfo +{ + // The FFmpeg name, which is what gets stored and what goes on the wire. + const char *name; + // What the settings dialog shows for it. + const char *displayName; +}; + +// Video codecs offered by the "Video Codec" setting, in the order they are +// listed. Every one of them is a codec the 0x75 message can name and that the +// receiving side hands to avcodec_find_decoder_by_name(), so no peer needs to +// know in advance which one we picked. +extern const CallCodecInfo kCallVideoCodecs[]; +extern const int kCallVideoCodecCount; + +// Audio codecs offered by the "Audio Codec" setting - the three the protocol +// names for the 0x74 message. +extern const CallCodecInfo kCallAudioCodecs[]; +extern const int kCallAudioCodecCount; + +// What a call uses when nothing has ever been configured: the values that were +// compiled in before any of this became a setting, so an install that has +// never opened the settings dialog behaves exactly as it did. +const int kDefaultVideoBitrateKbps = 600; +const int kDefaultAudioBitrateKbps = 32; +const int kDefaultCaptureWidth = 640; +const int kDefaultCaptureHeight = 480; +const int kDefaultCaptureFrameRate = 15; + +struct CallSettings +{ + // FFmpeg codec names, e.g. "h264" and "opus". + QString videoCodec; + QString audioCodec; + + // As the user enters them, in kbit/s. + int videoBitrateKbps; + int audioBitrateKbps; + + // What the camera is asked for. Not necessarily what it delivers - see + // VideoDevice::Capture(), which loosens these a step at a time rather than + // failing when a camera will not honour them. + int captureWidth; + int captureHeight; + int captureFrameRate; + + // Reads the stored settings, filling in the defaults above for anything + // that was never set. Must be called from a thread that may touch the + // settings database - not from a capture or playback loop, see + // AudioDevice::SetLinuxDriverName() for why. + static CallSettings load(); +}; + +// The rate a given audio codec is captured and encoded at, and announced at in +// the 0x74 message. +// +// It is a property of the codec rather than a setting: G.722 is defined at +// 16kHz and nowhere else, and A-law is a telephony codec whose whole point is +// 8kHz. Opus would take any of them and uses 16kHz for the reason spelled out +// in callsettings.cpp. +int callAudioSampleRate(const QString &codec); + +// Whether the codec's bitrate is the caller's to choose. Opus is; G.722 and +// A-law each pack a fixed number of bits per sample, so their bitrate follows +// from the sample rate and nothing can be asked of the encoder. +bool callAudioBitrateAdjustable(const QString &codec); + +// The bitrate a fixed rate codec ends up at, in kbit/s, for showing in the +// settings dialog. Meaningless for a codec callAudioBitrateAdjustable() is +// true for, which returns the configured value instead. +int callAudioFixedBitrateKbps(const QString &codec); + +#endif // CALLSETTINGS_H diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index 8685936..15c7c96 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -19,25 +19,28 @@ #include "../globals.h" #include "../audiodevice.h" #include "../videodevice.h" +#include "../callsettings.h" #include "../../PICA_netconf.h" #include "../../PICA_proto.h" #include <QGroupBox> #include <QVBoxLayout> #include <QHBoxLayout> +#include <QFormLayout> #include <QLabel> #include <QDebug> #include <QVariant> #include <QListWidget> #include <QNetworkInterface> #include <QHostAddress> +#include <QSize> #include <QTabWidget> #include <QTimer> -// Audio pipeline test: how long the microphone is recorded for, and the -// sample rate to record at - the same rate a call negotiates. +// Audio pipeline test: how long the microphone is recorded for. The codec and +// the rate come from the "Call Settings" tab, so the test runs the same path a +// call would - see toggleAudioTest(). static const int kAudioTestRecordMs = 5000; -static const int kAudioTestSampleRate = 48000; // One encoded packet covers this much audio (the Opus encoder produces // 20 ms frames at 48 kHz), so feeding one packet per this many milliseconds @@ -55,14 +58,41 @@ static const int kAudioTestStartTimeoutMs = 4000; // buffer and the sound card's own buffer to drain before closing the device. static const int kAudioTestDrainMs = 500; -// Size requested from the camera by the video pipeline test, matching what a -// call asks for. -static const int kVideoTestWidth = 640; -static const int kVideoTestHeight = 480; +// Sizes and frame rates offered when the camera will not say what it supports +// - a platform where they cannot be queried, or a camera that will not answer. +// The camera is free to refuse any of them, the same way it may refuse +// anything else it was asked for; VideoDevice::Capture() loosens the request +// rather than failing. +static const struct +{ + int width; + int height; +} kFallbackResolutions[] = +{ + { 1920, 1080 }, { 1280, 720 }, { 800, 600 }, { 640, 480 }, + { 640, 360 }, { 352, 288 }, { 320, 240 }, { 160, 120 } +}; + +static const int kFallbackFrameRates[] = { 30, 25, 24, 20, 15, 10, 5 }; + +// Range the bitrate spin boxes accept, in kbit/s. Wide enough not to argue +// with anyone: the low end is where a codec still produces something, the high +// end well past what a call over a home connection can carry. +static const int kMinVideoBitrateKbps = 50; +static const int kMaxVideoBitrateKbps = 20000; +static const int kMinAudioBitrateKbps = 6; +static const int kMaxAudioBitrateKbps = 510; SettingsDialog::SettingsDialog(QWidget *parent) : QDialog(parent) { + // The video tab's format lists follow the camera selection, and + // fillVideoDevices() below changes that before they have been built - so + // they have to be readable as "not there yet" from the moment anything can + // reach them. + videoRes = nullptr; + videoFps = nullptr; + QVBoxLayout *settingsLayout = new QVBoxLayout(); QTabWidget *tabW = new QTabWidget(this); @@ -71,11 +101,13 @@ SettingsDialog::SettingsDialog(QWidget *parent) : QWidget* multilogintab = new QWidget(0); QWidget* audiodevtab = new QWidget(0); QWidget* videodevtab = new QWidget(0); + QWidget* calltab = new QWidget(0); QVBoxLayout *directc2cLayout = new QVBoxLayout(); QVBoxLayout *multiloginlayout = new QVBoxLayout(); QVBoxLayout *audiodevlayout = new QVBoxLayout(); QVBoxLayout *videodevLayout = new QVBoxLayout(); + QVBoxLayout *callLayout = new QVBoxLayout(); //Direct connections rbDisableDirectConns = new QRadioButton(tr("Disable direct connections"), this); @@ -305,6 +337,29 @@ SettingsDialog::SettingsDialog(QWidget *parent) : videoDevRefresh = new QPushButton(tr("Refresh 🔄"), this); connect(videoDevRefresh, SIGNAL(clicked()), this, SLOT(fillVideoDevices())); + QLabel *lbVideoRes = new QLabel(tr("Resolution"), this); + videoRes = new QComboBox(this); + videoRes->setToolTip(tr("Picture sizes the selected camera offers. A size the camera can " + "deliver already compressed lists the formats in brackets; those " + "are the ones the \"prefer compressed formats\" setting below can " + "forward untouched, without decoding and re-encoding them here.")); + + QLabel *lbVideoFps = new QLabel(tr("Frame rate"), this); + videoFps = new QComboBox(this); + videoFps->setToolTip(tr("Frame rates the camera offers at the selected size.")); + + // Filled from the camera the device list above has selected, so the two + // have to follow it - and each other. + fillVideoResolutions(); + connect(videoDev, SIGNAL(currentIndexChanged(int)), this, SLOT(fillVideoResolutions())); + connect(videoRes, SIGNAL(currentIndexChanged(int)), this, SLOT(fillVideoFrameRates())); + + QHBoxLayout *videoFormatLayout = new QHBoxLayout(); + videoFormatLayout->addWidget(lbVideoRes); + videoFormatLayout->addWidget(videoRes, 1); + videoFormatLayout->addWidget(lbVideoFps); + videoFormatLayout->addWidget(videoFps); + cbPreferCompressed = new QCheckBox(tr("Prefer compressed formats if provided by the camera"), this); #ifdef HAVE_VAAPI @@ -336,6 +391,7 @@ SettingsDialog::SettingsDialog(QWidget *parent) : videodevLayout->addWidget(videoDev); videodevLayout->addWidget(videoDevRefresh); + videodevLayout->addLayout(videoFormatLayout); videodevLayout->addWidget(cbPreferCompressed); #ifdef HAVE_VAAPI videodevLayout->addWidget(cbVaapiEncoding); @@ -376,10 +432,73 @@ SettingsDialog::SettingsDialog(QWidget *parent) : #endif testDecoderThread.start(); +//Call Settings + // What a call encodes with. The codec names stored here are FFmpeg names, + // which is also how the 0x74 and 0x75 messages name them - see + // callsettings.h. + callVideoCodec = new QComboBox(this); + + for (int i = 0; i < kCallVideoCodecCount; i++) + callVideoCodec->addItem(QLatin1String(kCallVideoCodecs[i].displayName), + QLatin1String(kCallVideoCodecs[i].name)); + + callVideoCodec->setToolTip(tr("What outgoing video is encoded with. The peer is told which " + "codec is in use, so it does not have to be one it was " + "configured for.\n\n" + "This applies to video encoded here. With \"prefer compressed " + "formats\" switched on and a camera that delivers a compressed " + "stream itself, that stream is forwarded as it comes and the " + "camera's format is what the peer receives - this setting then " + "only decides which of the camera's formats is preferred.")); + + callAudioCodec = new QComboBox(this); + + for (int i = 0; i < kCallAudioCodecCount; i++) + callAudioCodec->addItem(QLatin1String(kCallAudioCodecs[i].displayName), + QLatin1String(kCallAudioCodecs[i].name)); + + callAudioCodec->setToolTip(tr("What outgoing audio is encoded with. Opus is the one to want " + "for a voice call; G.722 and G.711 A-law are the telephony " + "codecs, fixed at 64 kbit/s and there for compatibility.")); + + callVideoBitrate = new QSpinBox(this); + callVideoBitrate->setRange(kMinVideoBitrateKbps, kMaxVideoBitrateKbps); + callVideoBitrate->setSuffix(tr(" kbps")); + callVideoBitrate->setValue(kDefaultVideoBitrateKbps); + + callAudioBitrate = new QSpinBox(this); + callAudioBitrate->setRange(kMinAudioBitrateKbps, kMaxAudioBitrateKbps); + callAudioBitrate->setSuffix(tr(" kbps")); + callAudioBitrate->setValue(kDefaultAudioBitrateKbps); + audioBitrateKbps = kDefaultAudioBitrateKbps; + + connect(callAudioCodec, SIGNAL(currentIndexChanged(int)), this, SLOT(callAudioCodecChanged())); + + QFormLayout *callFormLayout = new QFormLayout(); + callFormLayout->addRow(tr("Video Codec"), callVideoCodec); + callFormLayout->addRow(tr("Video Bitrate"), callVideoBitrate); + callFormLayout->addRow(tr("Audio Codec"), callAudioCodec); + callFormLayout->addRow(tr("Audio Bitrate"), callAudioBitrate); + + // The other half of what a call is started with lives on the video tab, + // next to the camera that has to deliver it; say so rather than leaving + // this tab looking like the whole story. + QLabel *lbCallHint = new QLabel(tr("The picture size and frame rate a call is started with are " + "on the Video Devices tab, next to the camera that has to " + "deliver them."), this); + lbCallHint->setWordWrap(true); + + callLayout->addLayout(callFormLayout); + callLayout->addWidget(lbCallHint); + callLayout->addStretch(1); + + calltab->setLayout(callLayout); + tabW->addTab(directc2ctab, tr("Direct Connections")); tabW->addTab(multilogintab, tr("Multiple logins")); tabW->addTab(audiodevtab, tr("Audio Devices")); tabW->addTab(videodevtab, tr("Video Devices")); + tabW->addTab(calltab, tr("Call Settings")); tabW->addTab(soundstab, tr("Sounds")); settingsLayout->addWidget(tabW); @@ -511,9 +630,16 @@ void SettingsDialog::toggleAudioTest() btAudioTest->setText(tr("Stop ⏹")); audioTestStatus->setText(tr("Recording, say something...")); + // The codec selected on the "Call Settings" tab, at the rate a call with + // it would run at - same as the video test, this exercises what the + // current selections do rather than a fixed pipeline of its own. + const QString codec = callAudioCodec->itemData(callAudioCodec->currentIndex()).toString(); + const int rate = callAudioSampleRate(codec); + const int bitrate = callAudioBitrateAdjustable(codec) ? callAudioBitrate->value() * 1000 : 0; + QMetaObject::invokeMethod(testMic, "configureCapture", Qt::QueuedConnection, - Q_ARG(QString, dev), Q_ARG(QString, QStringLiteral("opus")), - Q_ARG(int, kAudioTestSampleRate)); + Q_ARG(QString, dev), Q_ARG(QString, codec), + Q_ARG(int, rate), Q_ARG(int, bitrate)); QMetaObject::invokeMethod(testMic, "Capture", Qt::QueuedConnection); // Only a watchdog for the device failing to open at all. The recording @@ -589,9 +715,13 @@ void SettingsDialog::startAudioTestPlayback() audioTestStatus->setText(summary); + // Decoded with what it was encoded with, exactly as the receiving end of a + // call decodes with whatever the peer announced. + const QString codec = callAudioCodec->itemData(callAudioCodec->currentIndex()).toString(); + QMetaObject::invokeMethod(testSpeaker, "configurePlayback", Qt::QueuedConnection, - Q_ARG(QString, dev), Q_ARG(QString, QStringLiteral("opus")), - Q_ARG(int, kAudioTestSampleRate)); + Q_ARG(QString, dev), Q_ARG(QString, codec), + Q_ARG(int, callAudioSampleRate(codec))); QMetaObject::invokeMethod(testSpeaker, "Play", Qt::QueuedConnection); // The recording has to be handed over at the rate it plays at, not all at @@ -715,10 +845,16 @@ void SettingsDialog::toggleVideoTest() videoTestPathReport.clear(); + // The settings as they stand in the dialog, not as they were last stored: + // the point of the test is to see what the current selections do before + // committing to them. QMetaObject::invokeMethod(testCam, "configureCapture", Qt::QueuedConnection, Q_ARG(QString, dev), - Q_ARG(int, kVideoTestWidth), Q_ARG(int, kVideoTestHeight), + Q_ARG(int, selectedVideoWidth()), Q_ARG(int, selectedVideoHeight()), + Q_ARG(int, selectedVideoFrameRate()), Q_ARG(bool, cbPreferCompressed->isChecked()), + Q_ARG(QString, callVideoCodec->itemData(callVideoCodec->currentIndex()).toString()), + Q_ARG(int, callVideoBitrate->value() * 1000), Q_ARG(bool, vaapiEncoding)); QMetaObject::invokeMethod(testCam, "Capture", Qt::QueuedConnection); @@ -898,10 +1034,11 @@ void SettingsDialog::fillDevicesComboBox(QComboBox *cb, MediaDevice *dev, enum M .arg(md.at(i).device) .arg(md.at(i).humanReadable); - // Cameras that can deliver a compressed stream themselves say so, - // which is what the "prefer compressed formats" setting acts on. - if (!md.at(i).compressedFormats.isEmpty()) - item += QString(QLatin1String(" [%1]")).arg(md.at(i).compressedFormats.join(QLatin1String(", "))); + // A camera's compressed formats are not listed here but in the + // resolution list, which is where they actually mean something: a + // camera offers them at some of its sizes and not at others, so + // naming them against the camera as a whole said less than it + // appeared to. cb->addItem(item, md.at(i).device); } @@ -910,7 +1047,193 @@ void SettingsDialog::fillDevicesComboBox(QComboBox *cb, MediaDevice *dev, enum M void SettingsDialog::fillVideoDevices() { VideoDevice vd; + + // Refilling walks the current index through -1 and back, and each step + // of that would otherwise ask a camera what it can do - which is not + // free, on Windows it means instantiating the capture filter. So the + // format lists below are rebuilt once, at the end. + const QString previous = videoDev->itemData(videoDev->currentIndex()).toString(); + + videoDev->blockSignals(true); fillDevicesComboBox(videoDev, &vd, CAPTURE); + + // Refresh is for picking up a camera that was plugged in, not for + // moving the selection off the one already chosen. + int item = videoDev->findData(previous); + if (item >= 0) + videoDev->setCurrentIndex(item); + + videoDev->blockSignals(false); + + fillVideoResolutions(); +} + +void SettingsDialog::fillVideoResolutions() +{ + // Nothing to fill into yet while the constructor is still building the + // tab - fillVideoDevices() runs before these exist. + if (!videoRes || !videoFps) + return; + + const QString dev = videoDev->itemData(videoDev->currentIndex()).toString(); + + videoFormats = dev.isEmpty() ? QList<VideoCaptureFormat>() + : VideoDevice::CaptureFormats(dev); + + // Refilling moves the current index about; the frame rate list is rebuilt + // once at the end rather than on every one of those intermediate changes. + videoRes->blockSignals(true); + videoRes->clear(); + + if (videoFormats.isEmpty()) + { + // Either the camera would not say or this platform cannot ask. Offer + // the common sizes rather than an empty list - a camera that cannot + // produce one of them is handled the same way as any other request it + // will not honour, see VideoDevice::Capture(). + for (unsigned int i = 0; i < sizeof(kFallbackResolutions) / sizeof(kFallbackResolutions[0]); i++) + { + const int w = kFallbackResolutions[i].width; + const int h = kFallbackResolutions[i].height; + + videoRes->addItem(QString(QLatin1String("%1x%2")).arg(w).arg(h), QSize(w, h)); + } + } + else + { + for (int i = 0; i < videoFormats.size(); i++) + { + const VideoCaptureFormat &f = videoFormats.at(i); + QString item = QString(QLatin1String("%1x%2")).arg(f.width).arg(f.height); + + // The formats this camera can hand over ready-made at this size, + // which is what "prefer compressed formats" below acts on. + if (!f.compressedFormats.isEmpty()) + item += QString(QLatin1String(" [%1]")).arg(f.compressedFormats.join(QLatin1String(", "))); + + videoRes->addItem(item, QSize(f.width, f.height)); + } + } + + // Keep the configured size selected across a device change where the new + // camera offers it too, rather than silently moving the setting to + // whatever happens to be first in the new list. + Settings st(config_dbname); + QSize wanted(st.loadValue("video.capture_width", kDefaultCaptureWidth).toInt(), + st.loadValue("video.capture_height", kDefaultCaptureHeight).toInt()); + + int item = videoRes->findData(wanted); + + if (item < 0) + item = videoRes->findData(QSize(kDefaultCaptureWidth, kDefaultCaptureHeight)); + + if (item >= 0) + videoRes->setCurrentIndex(item); + + videoRes->blockSignals(false); + + fillVideoFrameRates(); +} + +void SettingsDialog::fillVideoFrameRates() +{ + if (!videoFps) + return; + + const QSize size = videoRes->itemData(videoRes->currentIndex()).toSize(); + QList<int> rates; + + for (int i = 0; i < videoFormats.size(); i++) + { + if (videoFormats.at(i).width == size.width() && videoFormats.at(i).height == size.height()) + { + rates = videoFormats.at(i).frameRates; + break; + } + } + + videoFps->blockSignals(true); + videoFps->clear(); + + if (rates.isEmpty()) + { + // Same reasoning as for the sizes: a camera that named its sizes but + // not its rates, or one we could not ask at all. + for (unsigned int i = 0; i < sizeof(kFallbackFrameRates) / sizeof(kFallbackFrameRates[0]); i++) + rates << kFallbackFrameRates[i]; + } + + for (int i = 0; i < rates.size(); i++) + videoFps->addItem(tr("%1 fps").arg(rates.at(i)), rates.at(i)); + + Settings st(config_dbname); + int wanted = st.loadValue("video.capture_framerate", kDefaultCaptureFrameRate).toInt(); + + int item = videoFps->findData(wanted); + + if (item < 0) + item = videoFps->findData(kDefaultCaptureFrameRate); + + // Neither the configured rate nor the default is on offer at this size - + // take the highest the camera does offer, which is the first in the list. + if (item < 0 && videoFps->count() > 0) + item = 0; + + if (item >= 0) + videoFps->setCurrentIndex(item); + + videoFps->blockSignals(false); +} + +int SettingsDialog::selectedVideoWidth() const +{ + const QSize size = videoRes->itemData(videoRes->currentIndex()).toSize(); + + return size.isValid() ? size.width() : kDefaultCaptureWidth; +} + +int SettingsDialog::selectedVideoHeight() const +{ + const QSize size = videoRes->itemData(videoRes->currentIndex()).toSize(); + + return size.isValid() ? size.height() : kDefaultCaptureHeight; +} + +int SettingsDialog::selectedVideoFrameRate() const +{ + bool ok = false; + const int fps = videoFps->itemData(videoFps->currentIndex()).toInt(&ok); + + return (ok && fps > 0) ? fps : kDefaultCaptureFrameRate; +} + +void SettingsDialog::callAudioCodecChanged() +{ + const QString codec = callAudioCodec->itemData(callAudioCodec->currentIndex()).toString(); + + if (callAudioBitrateAdjustable(codec)) + { + // Coming back from a fixed rate codec, whose rate is showing in the + // box; the user's own choice is in audioBitrateKbps. + if (!callAudioBitrate->isEnabled()) + { + callAudioBitrate->setEnabled(true); + callAudioBitrate->setValue(audioBitrateKbps); + callAudioBitrate->setToolTip(QString()); + } + + return; + } + + if (callAudioBitrate->isEnabled()) + audioBitrateKbps = callAudioBitrate->value(); + + // Show what the codec will actually run at rather than leaving a number + // behind that has nothing to do with it. + callAudioBitrate->setEnabled(false); + callAudioBitrate->setValue(callAudioFixedBitrateKbps(codec)); + callAudioBitrate->setToolTip(tr("This codec codes a fixed number of bits per sample, so its " + "bitrate follows from the sample rate and cannot be chosen.")); } void SettingsDialog::audioDriverChanged(int index) @@ -1099,8 +1422,38 @@ void SettingsDialog::loadSettings() if (videoDevItem >= 0) videoDev->setCurrentIndex(videoDevItem); + // The resolution and frame rate lists pick the stored size and rate out of + // what the selected camera offers as they are filled, which the line above + // has just triggered where it changed the camera - see + // fillVideoResolutions(). + cbPreferCompressed->setChecked(st.loadValue("video.prefer_compressed", 0).toBool()); + // Call encoding parameters. Read through CallSettings rather than from st + // directly, so that the dialog and a call that is started without ever + // opening it agree on what "not configured" means. + const CallSettings cs = CallSettings::load(); + + int callVideoCodecItem = callVideoCodec->findData(cs.videoCodec); + if (callVideoCodecItem >= 0) + callVideoCodec->setCurrentIndex(callVideoCodecItem); + + callVideoBitrate->setValue(qBound(kMinVideoBitrateKbps, cs.videoBitrateKbps, kMaxVideoBitrateKbps)); + + // Before the codec is selected: selecting one whose bitrate is fixed puts + // that codec's rate in the box and keeps this as the value to come back + // to, see callAudioCodecChanged(). + audioBitrateKbps = qBound(kMinAudioBitrateKbps, cs.audioBitrateKbps, kMaxAudioBitrateKbps); + callAudioBitrate->setValue(audioBitrateKbps); + + int callAudioCodecItem = callAudioCodec->findData(cs.audioCodec); + if (callAudioCodecItem >= 0) + callAudioCodec->setCurrentIndex(callAudioCodecItem); + + // Explicitly, because setCurrentIndex() above only emits when the index + // actually changes - and the stored codec is often the one already showing. + callAudioCodecChanged(); + #ifdef HAVE_VAAPI cbVaapiEncoding->setChecked(st.loadValue("video.vaapi_encoding", 0).toBool()); cbVaapiDecoding->setChecked(st.loadValue("video.vaapi_decoding", 0).toBool()); @@ -1182,7 +1535,22 @@ void SettingsDialog::storeSettings() st.storeValue("multiple_logins.state", QString::number(mlpstate)); st.storeValue("video.capture_device", videoDev->itemData(videoDev->currentIndex()).toString()); + st.storeValue("video.capture_width", QString::number(selectedVideoWidth())); + st.storeValue("video.capture_height", QString::number(selectedVideoHeight())); + st.storeValue("video.capture_framerate", QString::number(selectedVideoFrameRate())); st.storeValue("video.prefer_compressed", cbPreferCompressed->isChecked() ? "1" : "0"); + + st.storeValue("call.video_codec", callVideoCodec->itemData(callVideoCodec->currentIndex()).toString()); + st.storeValue("call.audio_codec", callAudioCodec->itemData(callAudioCodec->currentIndex()).toString()); + st.storeValue("call.video_bitrate_kbps", QString::number(callVideoBitrate->value())); + + // The box shows a fixed rate codec's own bitrate while such a codec is + // selected; what gets stored is the user's choice either way, so that + // switching back to Opus finds it again on the next visit. + if (callAudioBitrate->isEnabled()) + audioBitrateKbps = callAudioBitrate->value(); + + st.storeValue("call.audio_bitrate_kbps", QString::number(audioBitrateKbps)); #ifdef HAVE_VAAPI st.storeValue("video.vaapi_encoding", cbVaapiEncoding->isChecked() ? "1" : "0"); st.storeValue("video.vaapi_decoding", cbVaapiDecoding->isChecked() ? "1" : "0"); diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h index ba04612..2167f5f 100644 --- a/pica-client/dialogs/settingsdialog.h +++ b/pica-client/dialogs/settingsdialog.h @@ -94,6 +94,32 @@ private: QComboBox *videoDev; QPushButton *videoDevRefresh; + // Picture size and frame rate to ask the selected camera for. Both are + // filled from what the camera says it can do - the sizes from the camera, + // the rates from the size selected next to them - so they are refilled + // whenever the selection above them changes. + QComboBox *videoRes; + QComboBox *videoFps; + + // What the selected camera reported, kept so that changing the resolution + // can refill the frame rates without asking the camera again. Enumerating + // is not free - on Windows it means instantiating the capture filter. + QList<VideoCaptureFormat> videoFormats; + + // Encoding parameters for a call, on the "Call Settings" tab. The codec + // combo boxes carry the FFmpeg codec name as item data, which is what gets + // stored and what goes into the 0x74/0x75 messages. + QComboBox *callVideoCodec; + QComboBox *callAudioCodec; + QSpinBox *callVideoBitrate; + QSpinBox *callAudioBitrate; + + // The audio bitrate the user chose, which is not always what + // callAudioBitrate shows: selecting a fixed rate codec puts that codec's + // rate in the box and disables it, and the chosen value has to survive + // that to come back when an adjustable codec is selected again. + int audioBitrateKbps; + // Local audio pipeline test: a few seconds of the selected microphone are // captured and encoded, then decoded again and played back through the // selected playback device - the same path a call takes, minus the @@ -184,12 +210,25 @@ private: void fillDevicesComboBox(QComboBox *cb, MediaDevice *dev, enum MediaDeviceStreamDirection dir); + // The picture size and frame rate the video tab currently has selected, + // used both by the local pipeline test and when the settings are stored. + int selectedVideoWidth() const; + int selectedVideoHeight() const; + int selectedVideoFrameRate() const; + private slots: void OK(); void Cancel(); void toggleIncomingConnections(bool checked); void toggleMultipleLogins(bool checked); void fillVideoDevices(); + // Asks the newly selected camera what it can do and refills the two lists + // below it. fillVideoFrameRates() alone when only the size changed. + void fillVideoResolutions(); + void fillVideoFrameRates(); + // Greys out the audio bitrate box for a codec whose bitrate is not ours to + // choose - see audioBitrateKbps. + void callAudioCodecChanged(); // Device names are driver specific, so all three lists have to be filled // again when the driver changes. Only ever connected on Linux. void audioDriverChanged(int index); diff --git a/pica-client/mediadevice.h b/pica-client/mediadevice.h index 542674b..932c8e8 100644 --- a/pica-client/mediadevice.h +++ b/pica-client/mediadevice.h @@ -16,10 +16,6 @@ struct MediaDeviceInfo QString device; QString humanReadable; int index; - // Compressed formats this device can deliver by itself, as FFmpeg codec - // names, most preferred first. Filled in for cameras that offer such a - // stream (see VideoDevice::CompressedFormats), empty otherwise. - QStringList compressedFormats; }; class MediaDevice diff --git a/pica-client/pica-client.pro b/pica-client/pica-client.pro index 279879a..4823c8a 100644 --- a/pica-client/pica-client.pro +++ b/pica-client/pica-client.pro @@ -48,6 +48,7 @@ SOURCES += main.cpp\ sound.cpp \ dialogs/settingsdialog.cpp \ settings.cpp \ + callsettings.cpp \ ../PICA_netconf.c \ ../PICA_signverify.c \ dialogs/nodesdialog.cpp \ @@ -93,6 +94,7 @@ HEADERS += mainwindow.h \ sound.h \ dialogs/settingsdialog.h \ settings.h \ + callsettings.h \ ../PICA_netconf.h \ ../PICA_security.h \ ../PICA_signverify.h \ diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp index 5b7d5fb..3fb468c 100644 --- a/pica-client/videodevice.cpp +++ b/pica-client/videodevice.cpp @@ -15,13 +15,18 @@ along with this program. If not, see <https://www.gnu.org/licenses/>. */ #include "videodevice.h" +#include "callsettings.h" #include "../PICA_proto.h" #include "../PICA_media.h" #include <QFile> #include <QDebug> #include <QMutexLocker> +#include <QMap> +#include <QPair> #include <cstring> +#include <algorithm> +#include <functional> #ifdef Q_OS_LINUX #include <linux/videodev2.h> @@ -54,11 +59,6 @@ extern "C" { #endif } -// Encoding defaults. Not exposed as settings yet - see the deferred list in -// the feature plan; a fixed, widely supported set of parameters for now. -static const int kFrameRate = 15; -static const int64_t kBitrate = 600000; - // Largest amount of encoded data one 0x77 message can carry over a c2c or // directc2c connection: the protocol's per-message payload cap, minus the // sequence number + timestamp header that every call media packet carries. @@ -106,6 +106,131 @@ static const struct }; #endif +// Which FFmpeg encoder produces each of the codecs the "Video Codec" setting +// offers, on the GPU and in software. Two names per codec because FFmpeg names +// its external library encoders after the library rather than after the codec, +// and the id as well so that a build without the library named here can still +// fall back to whatever other encoder it has for the same codec. +static const struct +{ + const char *codec; + AVCodecID id; + const char *vaapiEncoder; + const char *swEncoder; +} kVideoEncoders[] = +{ + { "h264", AV_CODEC_ID_H264, "h264_vaapi", "libx264" }, + { "hevc", AV_CODEC_ID_HEVC, "hevc_vaapi", "libx265" }, + { "vp9", AV_CODEC_ID_VP9, "vp9_vaapi", "libvpx-vp9" } +}; + +static int videoEncoderIndex(const QString &codec) +{ + for (unsigned int i = 0; i < sizeof(kVideoEncoders) / sizeof(kVideoEncoders[0]); i++) + { + if (codec == QLatin1String(kVideoEncoders[i].codec)) + return (int)i; + } + + // An unrecognised setting - written by a build offering something this one + // does not, or by hand - falls back to the codec every peer can decode + // rather than leaving the call without video. + return 0; +} + +// Sets the private options that make an encoder suitable for a live call: +// low latency, no frame reordering, and parameter sets the receiver can pick +// up mid-stream. Each of the three encoders spells all of that differently, +// which is why this is keyed on the encoder rather than on the codec. +static void applyLiveEncoderOptions(AVCodecContext *enc_ctx, const QString &encoderName, int sliceMaxSize) +{ + if (encoderName == QLatin1String("libx264")) + { + av_opt_set(enc_ctx->priv_data, "preset", "veryfast", 0); + av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0); + + // Keeps every slice within one datagram of a mediac2c connection, so + // that a lost fragment costs a slice of the picture rather than the + // whole frame. Harmless when the media goes over TCP instead - it + // only splits frames into more NAL units than strictly necessary. + av_opt_set(enc_ctx->priv_data, "x264opts", + QString("slice-max-size=%1").arg(sliceMaxSize).toUtf8().constData(), 0); + } + else if (encoderName == QLatin1String("libx265")) + { + av_opt_set(enc_ctx->priv_data, "preset", "ultrafast", 0); + av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0); + + // repeat-headers is the x265 equivalent of leaving + // AV_CODEC_FLAG_GLOBAL_HEADER off below: without it the VPS/SPS/PPS go + // out once, at the start, and a receiver that joins the stream later - + // which every receiver does, the call being already running by the + // time its decoder opens - never sees them. + // + // x265 has no slice-max-size; the nearest thing is a fixed number of + // slices per frame, which does not bound their size. Fragmentation + // handles oversized frames either way, a loss just costs more of the + // picture than it does with x264. + av_opt_set(enc_ctx->priv_data, "x265-params", "repeat-headers=1:bframes=0", 0); + } + else if (encoderName == QLatin1String("libvpx-vp9")) + { + // "realtime" with a high cpu-used is libvpx's zerolatency: it caps how + // long the encoder may spend per frame rather than letting it take as + // long as the quality target needs. + av_opt_set(enc_ctx->priv_data, "deadline", "realtime", 0); + av_opt_set_int(enc_ctx->priv_data, "cpu-used", 8, 0); + // No lookahead - it would hold frames back, which is latency in a + // live call - and keep the bitstream decodable across a loss. + av_opt_set_int(enc_ctx->priv_data, "lag-in-frames", 0, 0); + av_opt_set_int(enc_ctx->priv_data, "error-resilient", 1, 0); + + // VP9 carries no parameter sets to repeat: everything a decoder needs + // is in the keyframe itself, so joining mid-stream needs nothing + // special here. + } +} + +#ifdef Q_OS_WIN +// Defined in videodshow.cpp, alongside the enumeration functions - see the +// comment on pica_enumerate_dshow_video() for why the answers come from +// DirectShow rather than from FFmpeg's demuxer. +QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device); +#endif + +#if defined(Q_OS_LINUX) || defined(Q_OS_WIN) +// Frame rates worth offering out of a camera that reports a continuous range +// rather than a list of what it supports. Such a camera will take anything in +// between, so this is only a question of what to put in a dropdown. +static const int kStandardFrameRates[] = { 60, 50, 30, 25, 24, 20, 15, 10, 5 }; + +// Sizes offered for the same reason, out of a camera that reports a range of +// sizes rather than a list. Ones outside the range, or that the reported step +// does not land on, are dropped by the caller. +static const struct +{ + int width; + int height; +} kStandardSizes[] = +{ + { 1920, 1080 }, { 1280, 720 }, { 1024, 768 }, { 800, 600 }, + { 848, 480 }, { 640, 480 }, { 640, 360 }, { 424, 240 }, + { 352, 288 }, { 320, 240 }, { 176, 144 }, { 160, 120 } +}; + +// Largest picture first, and highest frame rate first within each. +static bool captureFormatLessThan(const VideoCaptureFormat &a, const VideoCaptureFormat &b) +{ + const qint64 areaA = (qint64)a.width * a.height; + const qint64 areaB = (qint64)b.width * b.height; + + if (areaA != areaB) + return areaA > areaB; + + return a.width > b.width; +} +#endif + #ifdef HAVE_VAAPI // Picks the GPU pixel format out of what the decoder offers. Returning the // first entry instead - which is what happens when VAAPI is not among them - @@ -166,10 +291,211 @@ QStringList VideoDevice::CompressedFormats(const QString &device) return result; } +#ifdef Q_OS_LINUX +// Every picture size the camera offers in one pixel format. +// +// A camera answers this in one of two ways: a list of the sizes it has, or a +// range it will scale anything within. There is no list to show for the +// second, so the standard sizes that fall inside the range stand in for one. +static QList<QPair<int, int> > v4l2FrameSizes(int fd, unsigned int pixelformat) +{ + QList<QPair<int, int> > sizes; + struct v4l2_frmsizeenum fs; + + memset(&fs, 0, sizeof fs); + fs.pixel_format = pixelformat; + + for (fs.index = 0; ::ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &fs) == 0; fs.index++) + { + if (fs.type == V4L2_FRMSIZE_TYPE_DISCRETE) + { + sizes << qMakePair((int)fs.discrete.width, (int)fs.discrete.height); + continue; + } + + for (unsigned int i = 0; i < sizeof(kStandardSizes) / sizeof(kStandardSizes[0]); i++) + { + const unsigned int w = kStandardSizes[i].width; + const unsigned int h = kStandardSizes[i].height; + + if (w < fs.stepwise.min_width || w > fs.stepwise.max_width || + h < fs.stepwise.min_height || h > fs.stepwise.max_height) + continue; + + // A continuous camera reports a step of one, so this only ever + // excludes anything on a genuinely stepwise one. + if (fs.stepwise.step_width > 1 && (w - fs.stepwise.min_width) % fs.stepwise.step_width) + continue; + + if (fs.stepwise.step_height > 1 && (h - fs.stepwise.min_height) % fs.stepwise.step_height) + continue; + + sizes << qMakePair((int)w, (int)h); + } + + // A non-discrete answer is a single entry describing the whole range; + // there is no index 1 to ask for. + break; + } + + return sizes; +} + +// v4l2 reports frame intervals - seconds per frame - and we want the rate, so +// every one of these is a reciprocal. +static double v4l2IntervalToFps(const struct v4l2_fract &f) +{ + if (f.numerator == 0 || f.denominator == 0) + return 0.0; + + return (double)f.denominator / (double)f.numerator; +} + +// The frame rates the camera offers for one pixel format at one size, highest +// first. Same two shapes of answer as the sizes above, handled the same way. +static QList<int> v4l2FrameRates(int fd, unsigned int pixelformat, int width, int height) +{ + QList<int> rates; + struct v4l2_frmivalenum fi; + + memset(&fi, 0, sizeof fi); + fi.pixel_format = pixelformat; + fi.width = width; + fi.height = height; + + for (fi.index = 0; ::ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &fi) == 0; fi.index++) + { + if (fi.type == V4L2_FRMIVAL_TYPE_DISCRETE) + { + int fps = qRound(v4l2IntervalToFps(fi.discrete)); + + if (fps > 0 && !rates.contains(fps)) + rates << fps; + + continue; + } + + // The longest interval is the lowest rate, hence the crossed over + // names here. + const double minFps = v4l2IntervalToFps(fi.stepwise.max); + const double maxFps = v4l2IntervalToFps(fi.stepwise.min); + + for (unsigned int i = 0; i < sizeof(kStandardFrameRates) / sizeof(kStandardFrameRates[0]); i++) + { + const int fps = kStandardFrameRates[i]; + + if (fps >= minFps && fps <= maxFps && !rates.contains(fps)) + rates << fps; + } + + break; + } + + std::sort(rates.begin(), rates.end(), std::greater<int>()); + + return rates; +} +#endif + +QList<VideoCaptureFormat> VideoDevice::CaptureFormats(const QString &device) +{ + QList<VideoCaptureFormat> result; + +#ifdef Q_OS_LINUX + int fd = ::open(device.toLatin1().constData(), O_RDWR | O_NONBLOCK); + if (fd < 0) + return result; + + // Keyed on the size, because a camera reports its sizes once per pixel + // format and the same size usually comes back several times over. What + // differs between those is the format and the frame rates available in + // it, so both are merged into the one entry for the size. + QMap<QPair<int, int>, VideoCaptureFormat> bySize; + + struct v4l2_fmtdesc fmt; + + memset(&fmt, 0, sizeof fmt); + fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + for (fmt.index = 0; ::ioctl(fd, VIDIOC_ENUM_FMT, &fmt) == 0; fmt.index++) + { + // What this pixel format is called in a 0x75 message, empty for an + // uncompressed one and for a compressed one we would not forward. + QString codec; + + for (unsigned int i = 0; i < sizeof(kCompressedFormatPreference) / sizeof(kCompressedFormatPreference[0]); i++) + { + if (kCompressedFormatPreference[i].v4l2_pixelformat == fmt.pixelformat) + { + codec = QLatin1String(kCompressedFormatPreference[i].ffmpeg_codec); + break; + } + } + + QList<QPair<int, int> > sizes = v4l2FrameSizes(fd, fmt.pixelformat); + + for (int s = 0; s < sizes.size(); s++) + { + VideoCaptureFormat &entry = bySize[sizes.at(s)]; + + entry.width = sizes.at(s).first; + entry.height = sizes.at(s).second; + + if (!codec.isEmpty() && !entry.compressedFormats.contains(codec)) + entry.compressedFormats << codec; + + QList<int> rates = v4l2FrameRates(fd, fmt.pixelformat, entry.width, entry.height); + + for (int r = 0; r < rates.size(); r++) + if (!entry.frameRates.contains(rates.at(r))) + entry.frameRates << rates.at(r); + } + } + + ::close(fd); + + for (QMap<QPair<int, int>, VideoCaptureFormat>::iterator it = bySize.begin(); it != bySize.end(); ++it) + { + VideoCaptureFormat entry = it.value(); + + // The formats were collected in whatever order the camera listed them + // in; report them in preference order, the same one + // CompressedFormats() and VideoDevice::Capture() work in. + QStringList ordered; + + for (unsigned int i = 0; i < sizeof(kCompressedFormatPreference) / sizeof(kCompressedFormatPreference[0]); i++) + { + QString codec = QLatin1String(kCompressedFormatPreference[i].ffmpeg_codec); + + if (entry.compressedFormats.contains(codec) && !ordered.contains(codec)) + ordered << codec; + } + + entry.compressedFormats = ordered; + + std::sort(entry.frameRates.begin(), entry.frameRates.end(), std::greater<int>()); + + result << entry; + } + + std::sort(result.begin(), result.end(), captureFormatLessThan); +#elif defined(Q_OS_WIN) + result = pica_dshow_capture_formats(device); +#else + Q_UNUSED(device); +#endif + + return result; +} + VideoFrameAssembler::VideoFrameAssembler() - : m_timestamp(0), m_inProgress(false), m_lastSeq(0), - m_lastWasFrameEnd(false), m_haveLastSeq(false) + : m_timestamp(0), m_lastSeq(0) { + // Through reset() rather than an initialiser list of its own: a fresh + // assembler and one that has been reset are the same thing, and stating + // that start-of-stream state twice is how the two came to disagree - the + // constructor's copy is what the first call after launch actually uses. + reset(); } void VideoFrameAssembler::reset() @@ -177,9 +503,24 @@ void VideoFrameAssembler::reset() m_buffer.clear(); m_inProgress = false; m_haveLastSeq = false; - // The first fragment received is as likely to be the middle of a frame as - // its beginning, so assembly only starts after the first frame boundary. - m_lastWasFrameEnd = false; + + // The next fragment is taken to begin a frame, rather than to be the + // middle of one whose start was lost. + // + // Every caller resets a stream that is about to start, never one being + // joined in flight - a call resets before the first packet can arrive, and + // the switch to a mediac2c connection deliberately does not reset, since + // media keeps arriving over either transport for the whole call. So the + // first fragment after this really is the first one the sender sent. + // + // Assuming the opposite costs the stream's first frame, and that frame is + // the one that matters: with a camera's own compressed stream being + // forwarded untouched, it carries the parameter sets and the only keyframe + // the camera will produce for the next ten seconds, so losing it means ten + // seconds of black. If this assumption is ever wrong the cost is one + // truncated frame that the decoder rejects, and the next frame boundary + // resynchronizes as usual. + m_lastWasFrameEnd = true; } QByteArray VideoFrameAssembler::addFragment(quint16 seq_num, quint32 timestamp, const QByteArray &data) @@ -242,9 +583,11 @@ QByteArray VideoFrameAssembler::addFragment(quint16 seq_num, quint32 timestamp, } VideoDevice::VideoDevice(QObject *parent) - : QObject(parent), m_width(640), m_height(480), m_preferCompressed(false), + : QObject(parent), m_width(kDefaultCaptureWidth), m_height(kDefaultCaptureHeight), + m_frameRate(kDefaultCaptureFrameRate), m_bitrate(kDefaultVideoBitrateKbps * 1000), + m_preferCompressed(false), m_useVaapi(false), m_useVaapiRender(false), m_abort(0), - m_maxFragmentSize(kMaxFragmentSize) + m_maxFragmentSize(kMaxFragmentSize), m_decoderStarted(0) { } @@ -261,12 +604,16 @@ VideoDevice::~VideoDevice() Close(); } -void VideoDevice::configureCapture(QString deviceName, int width, int height, - bool preferCompressed, bool useVaapi) +void VideoDevice::configureCapture(QString deviceName, int width, int height, int frameRate, + bool preferCompressed, QString codec, int bitrate, + bool useVaapi) { m_deviceName = deviceName; m_width = width; m_height = height; + m_frameRate = frameRate > 0 ? frameRate : kDefaultCaptureFrameRate; + m_codec = codec; + m_bitrate = bitrate > 0 ? bitrate : kDefaultVideoBitrateKbps * 1000; m_preferCompressed = preferCompressed; m_useVaapi = useVaapi; } @@ -284,6 +631,9 @@ void VideoDevice::configurePlayback(QString codec, int width, int height, // A renderer that failed during an earlier call says nothing about this // one - it may well be a different window. m_hwRenderDisabled.storeRelaxed(0); + // A fresh decode session, so the startup queue depth applies again until + // this one's loop is consuming - see enqueueFrame(). + m_decoderStarted.storeRelaxed(0); } void VideoDevice::disableHardwareRendering() @@ -295,7 +645,10 @@ void VideoDevice::enqueueFrame(QByteArray encodedFrame) { QMutexLocker locker(&m_queueMutex); - while (m_frameQueue.size() >= kMaxQueueDepth) + // Deeper while the decoder is still starting up - see kStartupQueueDepth. + const int depth = m_decoderStarted.loadRelaxed() ? kMaxQueueDepth : kStartupQueueDepth; + + while (m_frameQueue.size() >= depth) m_frameQueue.dequeue(); m_frameQueue.enqueue(encodedFrame); @@ -414,8 +767,18 @@ void VideoDevice::Capture() QStringList attempts; if (m_preferCompressed) + { attempts = CompressedFormats(m_deviceName); + // The configured codec first when the camera can produce it itself: + // forwarding that one means the peer receives exactly what was asked + // for, rather than whichever compressed format the camera happened to + // rank highest. The rest keep their preference order behind it, since + // forwarding any of them still beats decoding and re-encoding. + if (attempts.removeAll(m_codec) > 0) + attempts.prepend(m_codec); + } + attempts << QString(); // Listing a format and being able to produce it at a given size and frame @@ -452,7 +815,7 @@ void VideoDevice::Capture() ret = openCamera(ifmt, deviceUtf8, kConstraints[c].size ? m_width : 0, kConstraints[c].size ? m_height : 0, - kConstraints[c].rate ? kFrameRate : 0, + kConstraints[c].rate ? m_frameRate : 0, attempts.at(a), &ifmt_ctx); if (ret >= 0) @@ -467,7 +830,7 @@ void VideoDevice::Capture() if (!kConstraints[c].size) asked = QStringLiteral("any size or rate"); else if (kConstraints[c].rate) - asked = QString("%1x%2 @%3").arg(m_width).arg(m_height).arg(kFrameRate); + asked = QString("%1x%2 @%3").arg(m_width).arg(m_height).arg(m_frameRate); else asked = QString("%1x%2 at any rate").arg(m_width).arg(m_height); @@ -604,7 +967,7 @@ void VideoDevice::runPassthroughLoop(AVFormatContext *ifmt_ctx, AVStream *in_st, // one that was asked for - and when the open had to fall back to letting the // camera choose, was not asked for at all. Encoding at a rate the frames do // not arrive at gives a stream whose timestamps disagree with reality. -static AVRational cameraFrameRate(AVStream *in_st) +static AVRational cameraFrameRate(AVStream *in_st, int fallbackRate) { AVRational fr = in_st->avg_frame_rate; @@ -612,14 +975,19 @@ static AVRational cameraFrameRate(AVStream *in_st) fr = in_st->r_frame_rate; if (fr.num <= 0 || fr.den <= 0) - fr = AVRational{ kFrameRate, 1 }; + fr = AVRational{ fallbackRate, 1 }; return fr; } void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) { - const AVRational frameRate = cameraFrameRate(in_st); + const AVRational frameRate = cameraFrameRate(in_st, m_frameRate); + + // Which encoder produces the configured codec. Looked up once: the GPU + // attempt and the software fallback below are two encoders for the same + // codec, so the peer is told the same thing either way. + const int encIdx = videoEncoderIndex(m_codec); // Two seconds between keyframes. A receiver that joins late or loses a // packet waits this long for a picture, so it is a latency figure as much @@ -657,7 +1025,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) #ifdef HAVE_VAAPI if (m_useVaapi && VaapiContext::isAvailable()) { - enc = avcodec_find_encoder_by_name("h264_vaapi"); + enc = avcodec_find_encoder_by_name(kVideoEncoders[encIdx].vaapiEncoder); AVBufferRef *hw_device_ref = enc ? VaapiContext::deviceRef() : nullptr; if (hw_device_ref) @@ -684,18 +1052,18 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) enc_ctx->pix_fmt = AV_PIX_FMT_VAAPI; enc_ctx->time_base = av_inv_q(frameRate); enc_ctx->framerate = frameRate; - enc_ctx->bit_rate = kBitrate; + enc_ctx->bit_rate = m_bitrate; enc_ctx->gop_size = gopSize; enc_ctx->max_b_frames = 0; enc_ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref); - // No preset/tune here: those are x264's own options and - // mean nothing to a VAAPI encoder. + // No preset/tune here: those belong to the software + // encoders and mean nothing to a VAAPI one. if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) >= 0) hardware = true; else - qWarning() << QString("VAAPI H.264 encoder would not open (%1), encoding in software") - .arg(ff_errstr(ret)); + qWarning() << QString("The VAAPI %1 encoder would not open (%2), encoding in software") + .arg(QLatin1String(kVideoEncoders[encIdx].codec), ff_errstr(ret)); } else { @@ -711,7 +1079,8 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) } else { - qWarning() << "FFmpeg has no h264_vaapi encoder, encoding in software"; + qWarning() << QString("FFmpeg has no %1 encoder, encoding in software") + .arg(QLatin1String(kVideoEncoders[encIdx].vaapiEncoder)); } if (!hardware) @@ -723,27 +1092,35 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) } #endif + // Named rather than taken from enc->name because the VAAPI branch above + // may have left a different encoder in enc; filled in below for the + // software path, which is the only one the private options apply to. + QString swEncoderName; + if (!hardware) { - enc = avcodec_find_encoder_by_name("libx264"); + enc = avcodec_find_encoder_by_name(kVideoEncoders[encIdx].swEncoder); if (!enc) - enc = avcodec_find_encoder(AV_CODEC_ID_H264); + enc = avcodec_find_encoder(kVideoEncoders[encIdx].id); enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr; if (!enc || !enc_ctx) { - QString msg = "H.264 encoder not available"; + QString msg = QString("No %1 encoder available").arg(QLatin1String(kVideoEncoders[encIdx].codec)); qWarning() << msg; emit errorOccurred(msg); + if (enc_ctx) avcodec_free_context(&enc_ctx); avcodec_free_context(&dec_ctx); return; } + swEncoderName = QLatin1String(enc->name); + enc_ctx->width = m_width; enc_ctx->height = m_height; enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P; enc_ctx->time_base = av_inv_q(frameRate); enc_ctx->framerate = frameRate; - enc_ctx->bit_rate = kBitrate; + enc_ctx->bit_rate = m_bitrate; enc_ctx->gop_size = gopSize; // B-frames reorder output, which would cost latency in a live call. enc_ctx->max_b_frames = 0; @@ -751,19 +1128,11 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) // Note the absence of AV_CODEC_FLAG_GLOBAL_HEADER: we want the parameter // sets repeated in-band with every keyframe, since the receiving side has // no out-of-band way to learn them and may start decoding mid-stream. - av_opt_set(enc_ctx->priv_data, "preset", "veryfast", 0); - av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0); - - // Keeps every slice within one datagram of a mediac2c connection, so - // that a lost fragment costs a slice of the picture rather than the - // whole frame. Harmless when the media goes over TCP instead - it - // only splits frames into more NAL units than strictly necessary. - av_opt_set(enc_ctx->priv_data, "x264opts", - QString("slice-max-size=%1").arg(kSliceMaxSize).toUtf8().constData(), 0); + applyLiveEncoderOptions(enc_ctx, swEncoderName, kSliceMaxSize); if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) < 0) { - QString msg = QString("Could not open H.264 encoder: %1").arg(ff_errstr(ret)); + QString msg = QString("Could not open the %1 encoder: %2").arg(swEncoderName, ff_errstr(ret)); qWarning() << msg; emit errorOccurred(msg); avcodec_free_context(&enc_ctx); @@ -772,8 +1141,9 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) } } - emit accelerationInUse(hardware ? QStringLiteral("GPU encoding (VAAPI)") - : QStringLiteral("CPU encoding (libx264)")); + emit accelerationInUse(hardware + ? QString("GPU encoding (VAAPI %1)").arg(QLatin1String(kVideoEncoders[encIdx].codec)) + : QString("CPU encoding (%1)").arg(swEncoderName)); // Everything the camera sends is scaled to the encoder's size and re-encoded, // so this is what the peer will receive regardless of what the camera @@ -910,7 +1280,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) { if (!loggedEncodeError) { - qWarning() << "H.264 encode failed"; + qWarning() << QString("%1 encode failed").arg(QLatin1String(avcodec_get_name(enc_ctx->codec_id))); loggedEncodeError = true; } continue; @@ -1011,6 +1381,12 @@ void VideoDevice::Play() continue; frameData = m_frameQueue.dequeue(); + + // The codec is open and the loop is consuming, so the backlog held + // for startup is no longer wanted: from here on the newest frame + // wins. Set under the queue mutex so enqueueFrame() cannot read it + // while this frame is being taken. + m_decoderStarted.storeRelaxed(1); } AVPacket *in_pkt = av_packet_alloc(); @@ -1027,7 +1403,7 @@ void VideoDevice::Play() { if (!loggedDecodeError) { - qWarning() << QString("H.264 decode failed: %1").arg(ff_errstr(ret)); + qWarning() << QString("%1 decode failed: %2").arg(m_codec, ff_errstr(ret)); loggedDecodeError = true; } continue; @@ -1211,7 +1587,6 @@ QList<MediaDeviceInfo> VideoDevice::Enumerate(enum MediaDeviceStreamDirection di if (!(cap.device_caps & V4L2_CAP_VIDEO_CAPTURE)) continue; d.index = index++; - d.compressedFormats = CompressedFormats(d.device); result << d; } diff --git a/pica-client/videodevice.h b/pica-client/videodevice.h index 58eaef9..fe218ec 100644 --- a/pica-client/videodevice.h +++ b/pica-client/videodevice.h @@ -63,6 +63,29 @@ private: bool m_haveLastSeq; }; +// One picture size a camera can deliver, and what it can deliver it in. +// Reported by VideoDevice::CaptureFormats() and shown by the settings dialog's +// resolution and frame rate lists. +struct VideoCaptureFormat +{ + int width; + int height; + + // Frame rates the camera offers at this size, in whole frames per second, + // highest first. Empty when the camera would not say - which is not the + // same as "none": the capture code can still open it and let the camera + // pick, see VideoDevice::Capture(). + QList<int> frameRates; + + // Compressed formats available at this size, as FFmpeg codec names, most + // preferred first. The same names and the same order as + // CompressedFormats() reports for the whole camera, but per size - a + // camera routinely offers MJPEG at its larger sizes only, which is exactly + // what the "prefer compressed formats" setting turns on and what makes the + // choice of size worth showing alongside. + QStringList compressedFormats; +}; + // Video counterpart of AudioDevice: a single instance is dedicated to one // direction - either capture+encode (Capture(), reading from a local camera) // or decode (Play(), fed from the network via enqueueFrame()). Each instance @@ -83,18 +106,31 @@ public: VideoDevice(QObject *parent = nullptr); ~VideoDevice(); + // width, height and frameRate are what the camera is asked for; a camera + // that will not honour them is opened with them loosened rather than not + // at all, see Capture(). + // // preferCompressed asks for the camera's own compressed stream to be // forwarded untouched when it offers one, saving the cost of decoding and // re-encoding every frame. The codec that ends up being used is only // known once the camera has been opened, and is reported by // captureStarted() - it is not necessarily the one requested here. // + // codec is the FFmpeg name of the codec to encode to - "h264", "hevc" or + // "vp9" - and bitrate the rate to encode at, in bits per second. Both + // apply to frames this class encodes itself; a camera stream being + // forwarded untouched is already encoded and neither has anything to act + // on. codec is preferred over the camera's other compressed formats when + // looking for one to forward, so choosing a codec the camera can produce + // itself gets that one rather than merely a compatible one. + // // useVaapi asks for the frames to be encoded on the GPU. It is a request, // not a guarantee: without VAAPI support built in, without usable // hardware, or if the hardware encoder will not open, capture falls back // to encoding in software. - Q_INVOKABLE void configureCapture(QString deviceName, int width, int height, - bool preferCompressed, bool useVaapi); + Q_INVOKABLE void configureCapture(QString deviceName, int width, int height, int frameRate, + bool preferCompressed, QString codec, int bitrate, + bool useVaapi); // useVaapi decodes on the GPU. useVaapiRender additionally keeps the // decoded frames there and emits them through hwFrameReady() instead of @@ -185,11 +221,20 @@ public: // are decoded and re-encoded as usual. static QStringList CompressedFormats(const QString &device); + // Every picture size the given camera can deliver, largest first, with the + // frame rates and compressed formats available at each. Empty on a + // platform where the camera cannot be asked - the caller then falls back + // to offering a set of common sizes, which the camera is free to refuse + // the same way it may refuse any other request. + static QList<VideoCaptureFormat> CaptureFormats(const QString &device); + private: QString m_deviceName; QString m_codec; int m_width; int m_height; + int m_frameRate; + int m_bitrate; bool m_preferCompressed; bool m_useVaapi; bool m_useVaapiRender; @@ -206,8 +251,27 @@ private: QMutex m_queueMutex; QWaitCondition m_queueCond; QQueue<QByteArray> m_frameQueue; + + // Steady state: the newest frame wins, because for live video showing the + // current picture matters more than showing every picture. static const int kMaxQueueDepth = 2; + // Until the decode loop has taken its first frame it is not behind - it is + // still opening the codec, which happens only after the peer's 0x75 has + // been processed, by which time its packets are already arriving. Applying + // the rule above during that window drops the stream's first keyframe and + // then the frames that follow it, which costs every picture until the next + // keyframe: with a camera's own stream forwarded untouched that can be ten + // seconds away. So a bounded backlog is held instead - a couple of seconds + // at a typical call frame rate - and the queue reverts to the depth above + // as soon as the decoder is actually consuming. + static const int kStartupQueueDepth = 30; + + // Whether the decode loop has taken a frame yet; see enqueueFrame(). Read + // and written from both the decoding thread and whichever thread delivers + // packets, hence atomic. + QAtomicInt m_decoderStarted; + // Sends one encoded frame out as one or more protocol sized fragments. void emitFragments(const unsigned char *data, int size); diff --git a/pica-client/videodshow.cpp b/pica-client/videodshow.cpp index 150e8a6..9b389e9 100644 --- a/pica-client/videodshow.cpp +++ b/pica-client/videodshow.cpp @@ -15,6 +15,7 @@ along with this program. If not, see <https://www.gnu.org/licenses/>. */ #include "mediadevice.h" +#include "videodevice.h" #include <QtGlobal> @@ -22,9 +23,13 @@ #include <QDebug> #include <QList> +#include <QMap> +#include <QPair> #include <QString> #include <QStringList> #include <QVector> +#include <algorithm> +#include <functional> // INITGUID before the DirectShow headers so CLSID_SystemDeviceEnum and the // rest get defined here rather than needing strmiids at link time. The GUIDs @@ -301,6 +306,232 @@ QStringList pica_dshow_compressed_formats(const QString &device) return result; } +// Frame rates worth offering out of a pin that reports a range of frame +// intervals rather than one fixed rate. Same list, and the same reasoning, as +// kStandardFrameRates in videodevice.cpp. +static const int kDshowStandardFrameRates[] = { 60, 50, 30, 25, 24, 20, 15, 10, 5 }; + +// DirectShow measures a frame interval in 100 nanosecond units, and we want +// the rate, so this is a reciprocal like the v4l2 one. +static double intervalToFps(LONGLONG interval) +{ + if (interval <= 0) + return 0.0; + + return 10000000.0 / (double)interval; +} + +// Largest picture first - the same order VideoDevice::CaptureFormats() reports +// on Linux, so the settings dialog's list reads the same on both. +static bool dshowFormatLessThan(const VideoCaptureFormat &a, const VideoCaptureFormat &b) +{ + const qint64 areaA = (qint64)a.width * a.height; + const qint64 areaB = (qint64)b.width * b.height; + + if (areaA != areaB) + return areaA > areaB; + + return a.width > b.width; +} + +// Every picture size the camera can deliver, with the frame rates and +// compressed formats available at each. +// +// The counterpart of the v4l2 VIDIOC_ENUM_FRAMESIZES/VIDIOC_ENUM_FRAMEINTERVALS +// walk in VideoDevice::CaptureFormats(), and it comes off the same +// IAMStreamConfig enumeration pica_dshow_compressed_formats() uses: each media +// type carries a size and a FourCC, and the VIDEO_STREAM_CONFIG_CAPS alongside +// it carries the range of frame intervals that size can be delivered at. +// +// The same caveat applies as there: this instantiates the capture filter, so +// an empty list can mean "cannot be opened at all" as much as "offers +// nothing". Either way the caller falls back to a set of common sizes. +QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device) +{ + QList<VideoCaptureFormat> result; + + if (device.isEmpty()) + return result; + + bool didInit = false; + HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (SUCCEEDED(hr)) + didInit = true; + else if (hr != RPC_E_CHANGED_MODE) + return result; + + IMoniker *moniker = findCamera(device); + if (!moniker) + { + if (didInit) + CoUninitialize(); + return result; + } + + IBaseFilter *filter = nullptr; + hr = moniker->BindToObject(nullptr, nullptr, IID_IBaseFilter, (void **)&filter); + moniker->Release(); + + if (FAILED(hr) || !filter) + { + qWarning() << "DirectShow: could not instantiate" << device + << "to ask what sizes it offers"; + if (didInit) + CoUninitialize(); + return result; + } + + // Keyed on the size, because a camera lists one media type per size and + // format combination and the same size comes back once per format. + QMap<QPair<int, int>, VideoCaptureFormat> bySize; + + IEnumPins *enumPins = nullptr; + if (SUCCEEDED(filter->EnumPins(&enumPins)) && enumPins) + { + IPin *pin = nullptr; + + while (enumPins->Next(1, &pin, nullptr) == S_OK) + { + PIN_DIRECTION dir; + IAMStreamConfig *config = nullptr; + + if (SUCCEEDED(pin->QueryDirection(&dir)) && dir == PINDIR_OUTPUT && + SUCCEEDED(pin->QueryInterface(IID_IAMStreamConfig, (void **)&config)) && config) + { + int count = 0, size = 0; + + if (SUCCEEDED(config->GetNumberOfCapabilities(&count, &size)) && + size >= (int)sizeof(VIDEO_STREAM_CONFIG_CAPS)) + { + QVector<quint8> caps(size); + + for (int i = 0; i < count; i++) + { + AM_MEDIA_TYPE *mt = nullptr; + if (FAILED(config->GetStreamCaps(i, &mt, caps.data())) || !mt) + continue; + + const BITMAPINFOHEADER *bih = nullptr; + // The nominal interval of this particular media type, + // as opposed to the range in the caps below. + LONGLONG avgTimePerFrame = 0; + + if (mt->formattype == FORMAT_VideoInfo && + mt->cbFormat >= sizeof(VIDEOINFOHEADER)) + { + const VIDEOINFOHEADER *vih = (const VIDEOINFOHEADER *)mt->pbFormat; + bih = &vih->bmiHeader; + avgTimePerFrame = vih->AvgTimePerFrame; + } + else if (mt->formattype == FORMAT_VideoInfo2 && + mt->cbFormat >= sizeof(VIDEOINFOHEADER2)) + { + const VIDEOINFOHEADER2 *vih = (const VIDEOINFOHEADER2 *)mt->pbFormat; + bih = &vih->bmiHeader; + avgTimePerFrame = vih->AvgTimePerFrame; + } + + if (!bih || bih->biWidth <= 0 || bih->biHeight == 0) + { + freeMediaType(mt); + continue; + } + + // A bottom-up DIB states its height negative; the + // picture is the same size either way. + const int width = (int)bih->biWidth; + const int height = bih->biHeight < 0 ? (int)-bih->biHeight : (int)bih->biHeight; + + VideoCaptureFormat &entry = bySize[qMakePair(width, height)]; + entry.width = width; + entry.height = height; + + const QString fourcc = fourccToString(bih->biCompression); + + for (unsigned int f = 0; f < sizeof(kDshowCompressedFormats) / sizeof(kDshowCompressedFormats[0]); f++) + { + if (fourcc != QLatin1String(kDshowCompressedFormats[f].fourcc)) + continue; + + QString codec = QLatin1String(kDshowCompressedFormats[f].ffmpeg_codec); + + if (!entry.compressedFormats.contains(codec)) + entry.compressedFormats << codec; + + break; + } + + const VIDEO_STREAM_CONFIG_CAPS *scc = (const VIDEO_STREAM_CONFIG_CAPS *)caps.constData(); + + // The shortest interval is the highest rate, hence the + // crossed over names. + const double maxFps = intervalToFps(scc->MinFrameInterval); + const double minFps = intervalToFps(scc->MaxFrameInterval); + + int nominal = qRound(intervalToFps(avgTimePerFrame)); + + if (nominal > 0 && !entry.frameRates.contains(nominal)) + entry.frameRates << nominal; + + // A pin that accepts a range of intervals will take + // anything within it, so there is no list to show - + // the standard rates inside the range stand in for one, + // exactly as on the v4l2 side. + for (unsigned int r = 0; r < sizeof(kDshowStandardFrameRates) / sizeof(kDshowStandardFrameRates[0]); r++) + { + const int fps = kDshowStandardFrameRates[r]; + + if (fps >= minFps && fps <= maxFps && !entry.frameRates.contains(fps)) + entry.frameRates << fps; + } + + freeMediaType(mt); + } + } + } + + if (config) + config->Release(); + pin->Release(); + } + + enumPins->Release(); + } + + filter->Release(); + + for (QMap<QPair<int, int>, VideoCaptureFormat>::iterator it = bySize.begin(); it != bySize.end(); ++it) + { + VideoCaptureFormat entry = it.value(); + + // Report the formats in our preference order rather than in the order + // the camera happened to list them, the same way + // pica_dshow_compressed_formats() does. + QStringList ordered; + + for (unsigned int f = 0; f < sizeof(kDshowCompressedFormats) / sizeof(kDshowCompressedFormats[0]); f++) + { + QString codec = QLatin1String(kDshowCompressedFormats[f].ffmpeg_codec); + + if (entry.compressedFormats.contains(codec) && !ordered.contains(codec)) + ordered << codec; + } + + entry.compressedFormats = ordered; + + std::sort(entry.frameRates.begin(), entry.frameRates.end(), std::greater<int>()); + + result << entry; + } + + std::sort(result.begin(), result.end(), dshowFormatLessThan); + + if (didInit) + CoUninitialize(); + + return result; +} + // Lists the cameras DirectShow knows about. // // Capture itself goes through FFmpeg's "dshow" demuxer rather than through @@ -413,11 +644,10 @@ QList<MediaDeviceInfo> pica_enumerate_dshow_video() d.index = index++; - // Left empty deliberately. compressedFormats drives the "prefer - // compressed formats" setting, and working out what a DirectShow - // camera can deliver means walking its output pin's media types - // through IAMStreamConfig - a good deal more work than the v4l2 call - // that fills this in on Linux, and not needed to get a picture. + // Nothing here about what the camera can deliver: that means walking + // its output pin's media types, which pica_dshow_capture_formats() + // does once a camera has actually been selected rather than for every + // camera on the machine. result << d; } |
