summaryrefslogtreecommitdiffstats
path: root/pica-client
diff options
context:
space:
mode:
authorAnton <anton@picapica.im>2026-09-26 19:00:40 +0400
committerAnton <anton@picapica.im>2026-09-26 19:00:40 +0400
commit76225a538e1e960eda9e31a34846f4bc62adef87 (patch)
treee1a62b1c2c55b5c6c1e319d3c6726da4f4f96f7f /pica-client
parentbc708f0cc6c01999a8e111d1c66c899106e91bc7 (diff)
downloadpica-pica-76225a538e1e960eda9e31a34846f4bc62adef87.tar.gz
pica-pica-76225a538e1e960eda9e31a34846f4bc62adef87.zip
pica-client: configurable resolutions, codecs, bitrates for a call
Diffstat (limited to 'pica-client')
-rw-r--r--pica-client/audiodevice.cpp136
-rw-r--r--pica-client/audiodevice.h9
-rw-r--r--pica-client/audiovideocallcontroller.cpp57
-rw-r--r--pica-client/callsettings.cpp124
-rw-r--r--pica-client/callsettings.h106
-rw-r--r--pica-client/dialogs/settingsdialog.cpp400
-rw-r--r--pica-client/dialogs/settingsdialog.h39
-rw-r--r--pica-client/mediadevice.h4
-rw-r--r--pica-client/pica-client.pro2
-rw-r--r--pica-client/videodevice.cpp467
-rw-r--r--pica-client/videodevice.h68
-rw-r--r--pica-client/videodshow.cpp240
12 files changed, 1519 insertions, 133 deletions
diff --git a/pica-client/audiodevice.cpp b/pica-client/audiodevice.cpp
index fdbeedc..fdfd449 100644
--- a/pica-client/audiodevice.cpp
+++ b/pica-client/audiodevice.cpp
@@ -15,6 +15,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "audiodevice.h"
+#include "callsettings.h"
#include "nativeaudio.h"
#include <QDebug>
@@ -68,9 +69,10 @@ extern "C" {
// TonePlayer's design and keeps the resampling/framing code below simple.
static const int kChannels = 1;
-// Target bitrate for the Opus voice encoder. Not exposed as a setting yet;
-// a reasonable fixed default for a single speech channel.
-static const int64_t kOpusBitrate = 32000;
+// Bitrate an encoder runs at when configureCapture() was given none, and the
+// one the "Audio Bitrate" setting defaults to: a reasonable rate for a single
+// speech channel of Opus.
+static const int64_t kOpusBitrate = kDefaultAudioBitrateKbps * 1000;
// The FFmpeg driver name used on Linux. Kept in a process wide variable
// rather than read from the settings database on demand, because the audio
@@ -140,6 +142,65 @@ static SwrContext *makeMonoResampler(AVSampleFormat inFmt, int inRate, AVSampleF
return swr;
}
+// The AVCodecID behind one of the names the 0x74 message carries.
+//
+// Those are FFmpeg *codec* names (see doc/proto-doc-latest.txt), which are not
+// always the name of an encoder or decoder for them: FFmpeg names each
+// implementation after itself, so the G.722 codec is "adpcm_g722" while the
+// only encoder and decoder for it are both called "g722". Going through the
+// codec descriptor rather than through the implementation name is what makes
+// the protocol's names work for all three.
+static AVCodecID callAudioCodecId(const QString &codec)
+{
+ const AVCodecDescriptor *desc = avcodec_descriptor_get_by_name(codec.toLatin1().constData());
+
+ if (!desc || desc->type != AVMEDIA_TYPE_AUDIO)
+ return AV_CODEC_ID_NONE;
+
+ return desc->id;
+}
+
+// The encoder for one of the call audio codecs. Only "opus" needs a word of
+// its own, because FFmpeg has two Opus encoders and libopus is both the better
+// one and the only one offering the interleaved 16 bit format the echo
+// canceller wants.
+static const AVCodec *findCallAudioEncoder(const QString &codec)
+{
+ if (codec == QLatin1String("opus"))
+ {
+ const AVCodec *enc = avcodec_find_encoder_by_name("libopus");
+
+ if (enc)
+ return enc;
+ }
+
+ const AVCodecID id = callAudioCodecId(codec);
+
+ // AV_CODEC_ID_NONE covers a name that is not a codec at all, and one that
+ // names a video codec - which would otherwise be opened as one here and
+ // produce something no audio decoder on the far side can read.
+ return id == AV_CODEC_ID_NONE ? nullptr : avcodec_find_encoder(id);
+}
+
+// The decoder counterpart, for whatever codec the peer announced. Resolved by
+// name for the same reason VideoDevice::Play() does it: the 0x74 message names
+// the codec the way FFmpeg does, so a peer that picks a different one of the
+// three needs no codec list kept in step here.
+static const AVCodec *findCallAudioDecoder(const QString &codec)
+{
+ if (codec == QLatin1String("opus"))
+ {
+ const AVCodec *dec = avcodec_find_decoder_by_name("libopus");
+
+ if (dec)
+ return dec;
+ }
+
+ const AVCodecID id = callAudioCodecId(codec);
+
+ return id == AV_CODEC_ID_NONE ? nullptr : avcodec_find_decoder(id);
+}
+
// Which sample format to run the Opus encoder in. Interleaved 16 bit is
// preferred over whatever the encoder lists first, because that is the format
// the echo canceller works in - taking it means the microphone samples can go
@@ -182,7 +243,7 @@ static AVSampleFormat chooseEncoderSampleFmt(const AVCodec *enc)
}
AudioDevice::AudioDevice(QObject *parent)
- : QObject(parent), m_sampleRate(48000), m_abort(0),
+ : QObject(parent), m_sampleRate(48000), m_bitrate(kOpusBitrate), m_abort(0),
m_clipCount(0), m_levelCount(0), m_levelPeak(0),
m_lastPlayedSeq(0), m_havePlayedSeq(false)
{
@@ -229,11 +290,12 @@ AudioDevice::~AudioDevice()
Close();
}
-void AudioDevice::configureCapture(QString deviceName, QString codec, int sampleRate)
+void AudioDevice::configureCapture(QString deviceName, QString codec, int sampleRate, int bitrate)
{
m_deviceName = deviceName;
m_codec = codec;
m_sampleRate = sampleRate;
+ m_bitrate = bitrate > 0 ? bitrate : kOpusBitrate;
}
void AudioDevice::configurePlayback(QString deviceName, QString codec, int sampleRate)
@@ -431,15 +493,14 @@ void AudioDevice::captureFFmpeg(const QString &driver)
return;
}
- const AVCodec *enc = avcodec_find_encoder_by_name("libopus");
- if (!enc)
- enc = avcodec_find_encoder(AV_CODEC_ID_OPUS);
+ const AVCodec *enc = findCallAudioEncoder(m_codec);
AVCodecContext *enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr;
if (!enc || !enc_ctx)
{
- QString msg = "Opus encoder not available";
+ QString msg = QString("No %1 encoder available").arg(m_codec);
qWarning() << msg;
emit errorOccurred(msg);
+ if (enc_ctx) avcodec_free_context(&enc_ctx);
avcodec_free_context(&dec_ctx);
avformat_close_input(&ifmt_ctx);
return;
@@ -447,19 +508,21 @@ void AudioDevice::captureFFmpeg(const QString &driver)
enc_ctx->sample_rate = m_sampleRate;
enc_ctx->sample_fmt = chooseEncoderSampleFmt(enc);
- enc_ctx->bit_rate = kOpusBitrate;
+ enc_ctx->bit_rate = m_bitrate;
enc_ctx->time_base = AVRational{1, m_sampleRate};
applyMonoLayout(enc_ctx);
// "voip" favors speech intelligibility over faithfulness and has lower
// algorithmic delay than the default "audio" mode - a better fit for a
// live call than for e.g. music. Only libopus exposes this private
- // option; harmless no-op on the native "opus" encoder fallback.
- av_opt_set(enc_ctx->priv_data, "application", "voip", 0);
+ // option; the other codecs have no private options at all, and a PCM
+ // encoder has no priv_data to pass here in the first place.
+ if (m_codec == QLatin1String("opus") && enc_ctx->priv_data)
+ av_opt_set(enc_ctx->priv_data, "application", "voip", 0);
if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) < 0)
{
- QString msg = QString("Could not open Opus encoder: %1").arg(ff_errstr(ret));
+ QString msg = QString("Could not open the %1 encoder: %2").arg(m_codec, ff_errstr(ret));
qWarning() << msg;
emit errorOccurred(msg);
avcodec_free_context(&enc_ctx);
@@ -481,7 +544,11 @@ void AudioDevice::captureFFmpeg(const QString &driver)
}
AVAudioFifo *fifo = av_audio_fifo_alloc(enc_ctx->sample_fmt, kChannels, 1);
- int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : 960;
+ // An encoder that takes any number of samples per packet - a PCM one -
+ // leaves this at zero and the packet size is ours to pick. 20ms is what
+ // the other two produce and what the playback side's jitter buffer is
+ // sized in, so it is what everything here already assumes.
+ int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : m_sampleRate / 50;
// The canceller only gets to work in place when the encoder is taking
// interleaved 16 bit, which libopus does. On the native encoder, whose
@@ -590,7 +657,7 @@ void AudioDevice::captureFFmpeg(const QString &driver)
}
else if (!loggedEncodeError)
{
- qWarning() << "Opus encode failed";
+ qWarning() << QString("%1 encode failed").arg(m_codec);
loggedEncodeError = true;
}
av_frame_free(&enc_frame);
@@ -638,13 +705,11 @@ void AudioDevice::captureNative(const QString &api)
emit deviceFormatInUse(backend->formatDescription());
- const AVCodec *enc = avcodec_find_encoder_by_name("libopus");
- if (!enc)
- enc = avcodec_find_encoder(AV_CODEC_ID_OPUS);
+ const AVCodec *enc = findCallAudioEncoder(m_codec);
AVCodecContext *enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr;
if (!enc || !enc_ctx)
{
- QString msg = "Opus encoder not available";
+ QString msg = QString("No %1 encoder available").arg(m_codec);
qWarning() << msg;
emit errorOccurred(msg);
if (enc_ctx) avcodec_free_context(&enc_ctx);
@@ -655,15 +720,18 @@ void AudioDevice::captureNative(const QString &api)
enc_ctx->sample_rate = m_sampleRate;
enc_ctx->sample_fmt = chooseEncoderSampleFmt(enc);
- enc_ctx->bit_rate = kOpusBitrate;
+ enc_ctx->bit_rate = m_bitrate;
enc_ctx->time_base = AVRational{1, m_sampleRate};
applyMonoLayout(enc_ctx);
- av_opt_set(enc_ctx->priv_data, "application", "voip", 0);
+
+ // See captureFFmpeg() for why this is guarded.
+ if (m_codec == QLatin1String("opus") && enc_ctx->priv_data)
+ av_opt_set(enc_ctx->priv_data, "application", "voip", 0);
int ret = avcodec_open2(enc_ctx, enc, nullptr);
if (ret < 0)
{
- QString msg = QString("Could not open Opus encoder: %1").arg(ff_errstr(ret));
+ QString msg = QString("Could not open the %1 encoder: %2").arg(m_codec, ff_errstr(ret));
qWarning() << msg;
emit errorOccurred(msg);
avcodec_free_context(&enc_ctx);
@@ -706,7 +774,8 @@ void AudioDevice::captureNative(const QString &api)
ec.clear();
}
- const int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : 960;
+ // Same reasoning as in captureFFmpeg().
+ const int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : m_sampleRate / 50;
QVector<qint16> pending;
QVector<qint16> scratch(frame_size * 4);
@@ -783,7 +852,7 @@ void AudioDevice::captureNative(const QString &api)
}
else if (!loggedEncodeError)
{
- qWarning() << "Opus encode failed";
+ qWarning() << QString("%1 encode failed").arg(m_codec);
loggedEncodeError = true;
}
@@ -884,16 +953,18 @@ void AudioDevice::playFFmpeg(const QString &driver)
return;
}
- // Only "opus" is understood by this slice - matches what
- // AudioVideoCallController always negotiates via SendAudioParams().
- const AVCodec *in_codec = avcodec_find_decoder(AV_CODEC_ID_OPUS);
+ // Whatever the peer announced in its 0x74 message, looked up by the name
+ // it used - the peer's choice of codec is its own and need not match ours.
+ const AVCodec *in_codec = findCallAudioDecoder(m_codec);
AVCodecContext *dec_ctx = in_codec ? avcodec_alloc_context3(in_codec) : nullptr;
if (dec_ctx)
{
+ // A PCM decoder is told nothing by the bitstream and has to be given
+ // both of these; the others work them out for themselves.
dec_ctx->sample_rate = m_sampleRate;
applyMonoLayout(dec_ctx);
}
- if (m_codec != QLatin1String("opus") || !in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0)
+ if (!in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0)
{
QString msg = QString("Unsupported call audio codec: %1").arg(m_codec);
qWarning() << msg;
@@ -945,7 +1016,7 @@ void AudioDevice::playFFmpeg(const QString &driver)
{
if (!loggedDecodeError)
{
- qWarning() << QString("Opus decode failed: %1").arg(ff_errstr(ret));
+ qWarning() << QString("%1 decode failed: %2").arg(m_codec, ff_errstr(ret));
loggedDecodeError = true;
}
continue;
@@ -1054,14 +1125,15 @@ void AudioDevice::playNative(const QString &api)
emit deviceFormatInUse(backend->formatDescription());
- const AVCodec *in_codec = avcodec_find_decoder(AV_CODEC_ID_OPUS);
+ // Same as in playFFmpeg(): the codec the peer announced, by its name.
+ const AVCodec *in_codec = findCallAudioDecoder(m_codec);
AVCodecContext *dec_ctx = in_codec ? avcodec_alloc_context3(in_codec) : nullptr;
if (dec_ctx)
{
dec_ctx->sample_rate = m_sampleRate;
applyMonoLayout(dec_ctx);
}
- if (m_codec != QLatin1String("opus") || !in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0)
+ if (!in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0)
{
QString msg = QString("Unsupported call audio codec: %1").arg(m_codec);
qWarning() << msg;
@@ -1110,7 +1182,7 @@ void AudioDevice::playNative(const QString &api)
{
if (!loggedDecodeError)
{
- qWarning() << QString("Opus decode failed: %1").arg(ff_errstr(ret));
+ qWarning() << QString("%1 decode failed: %2").arg(m_codec, ff_errstr(ret));
loggedDecodeError = true;
}
continue;
diff --git a/pica-client/audiodevice.h b/pica-client/audiodevice.h
index b98acf2..c1feb8a 100644
--- a/pica-client/audiodevice.h
+++ b/pica-client/audiodevice.h
@@ -47,7 +47,12 @@ public:
// device identifier as accepted by the driver named by
// PlatformDriverName() (e.g. an ALSA PCM name, a PulseAudio source name,
// a WASAPI endpoint id, a CoreAudio device UID), or "default".
- Q_INVOKABLE void configureCapture(QString deviceName, QString codec, int sampleRate);
+ //
+ // codec is the FFmpeg name of the codec, which is also the name the 0x74
+ // message carries - "opus", "adpcm_g722" or "pcm_alaw". bitrate is in bits
+ // per second and only reaches an encoder with a rate to control: G.722 and
+ // A-law code a fixed number of bits per sample and simply ignore it.
+ Q_INVOKABLE void configureCapture(QString deviceName, QString codec, int sampleRate, int bitrate);
Q_INVOKABLE void configurePlayback(QString deviceName, QString codec, int sampleRate);
// Give this device the call's echo canceller. The capture direction runs
@@ -173,6 +178,8 @@ private:
QString m_deviceName;
QString m_codec;
int m_sampleRate;
+ // Capture direction only; see configureCapture().
+ int m_bitrate;
QAtomicInt m_abort;
diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp
index 72b6fc2..9b4ae4d 100644
--- a/pica-client/audiovideocallcontroller.cpp
+++ b/pica-client/audiovideocallcontroller.cpp
@@ -18,32 +18,11 @@
#include "globals.h"
#include "skynet.h"
#include "settings.h"
+#include "callsettings.h"
#include <QByteArray>
#include <QDebug>
#include <QMetaObject>
-// Capture/encode resolution announced to the peer in the 0x75 message. Fixed
-// for now - a resolution setting is a planned follow-up, see VideoDevice for
-// the rest of the encoding parameters.
-static const quint16 kVideoWidth = 640;
-static const quint16 kVideoHeight = 480;
-
-// Rate we capture and encode at, announced to the peer in the 0x74 message.
-//
-// 16kHz rather than 48kHz because it is what the echo canceller wants. AEC3
-// does its filtering in the 0-8kHz band whatever it is handed; at 48kHz it
-// splits the signal into four bands, cancels in the lowest and gates the rest,
-// which is more work for a result that is no better on speech.
-//
-// Costs nothing worth having for speech: Opus codes this as wideband, an 8kHz
-// audio bandwidth, which is well past telephone quality and past where voice
-// intelligibility stops improving.
-//
-// The playback direction is NOT this constant - it follows whatever the peer
-// announces (see incoming_audio_params()), so a call with an older build that
-// still sends 48kHz keeps working.
-static const int kCallSampleRate = 16000;
-
AudioVideoCallController::AudioVideoCallController(QObject *parent)
: QObject(parent), callwindow(0), ringdevice(0), is_active(false),
@@ -174,6 +153,21 @@ void AudioVideoCallController::startAudioPipeline()
{
Settings st(config_dbname);
QString capDev = st.loadValue("audio.capture_device", "default").toString();
+ const CallSettings cs = CallSettings::load();
+
+ // What we capture and encode at, announced to the peer in the 0x74
+ // message. It follows from the codec - see callAudioSampleRate().
+ //
+ // The playback direction is NOT this: it follows whatever the peer
+ // announces in its own 0x74 (see incoming_audio_params()), so a call with
+ // a peer that picked a different codec or rate keeps working.
+ const int sampleRate = callAudioSampleRate(cs.audioCodec);
+
+ // G.722 and A-law spend a fixed number of bits per sample and have no rate
+ // to ask for; the setting only means anything to Opus.
+ const int bitrate = callAudioBitrateAdjustable(cs.audioCodec)
+ ? cs.audioBitrateKbps * 1000
+ : 0;
m_audioSeq = 0;
m_callClock.start();
@@ -188,8 +182,12 @@ void AudioVideoCallController::startAudioPipeline()
// below us is doing the job or nobody is, and either way running this as
// well would only take a second bite out of the speech - the devices are
// opened accordingly, see AudioDevice::captureNative().
+ //
+ // Built at the rate the codec runs at rather than at a fixed one, so that
+ // choosing a codec does not quietly switch cancellation off: the capture
+ // side drops a canceller whose rate does not match its own.
if (AudioDevice::EchoCancellation() == EchoCancellationOwn)
- m_echoCanceller = EchoCancellerPtr(new EchoCanceller(kCallSampleRate));
+ m_echoCanceller = EchoCancellerPtr(new EchoCanceller(sampleRate));
else
m_echoCanceller.clear();
@@ -205,10 +203,11 @@ void AudioVideoCallController::startAudioPipeline()
// encoding right away - the peer's decoder is only ready once it has
// processed this 0x74 message, but since the call is carried over TCP,
// packets sent immediately after are guaranteed to arrive after it.
- skynet->SendAudioParams(m_peer_id, QStringLiteral("opus"), kCallSampleRate);
+ skynet->SendAudioParams(m_peer_id, cs.audioCodec, (quint16)sampleRate);
QMetaObject::invokeMethod(microphone, "configureCapture", Qt::QueuedConnection,
- Q_ARG(QString, capDev), Q_ARG(QString, QStringLiteral("opus")), Q_ARG(int, kCallSampleRate));
+ Q_ARG(QString, capDev), Q_ARG(QString, cs.audioCodec),
+ Q_ARG(int, sampleRate), Q_ARG(int, bitrate));
QMetaObject::invokeMethod(microphone, "Capture", Qt::QueuedConnection);
// output is configured lazily, once the peer's own 0x74 (see
@@ -239,6 +238,7 @@ void AudioVideoCallController::startVideoPipeline()
{
Settings st(config_dbname);
QString camDev = st.loadValue("video.capture_device", QString()).toString();
+ const CallSettings cs = CallSettings::load();
// Unlike the audio devices, which fall back to the platform's "default"
// device, there is no such name for cameras - so with nothing configured
@@ -278,8 +278,11 @@ void AudioVideoCallController::startVideoPipeline()
// video_capture_started()).
QMetaObject::invokeMethod(cam, "configureCapture", Qt::QueuedConnection,
Q_ARG(QString, camDev),
- Q_ARG(int, kVideoWidth), Q_ARG(int, kVideoHeight),
- Q_ARG(bool, preferCompressed), Q_ARG(bool, vaapiEncoding));
+ Q_ARG(int, cs.captureWidth), Q_ARG(int, cs.captureHeight),
+ Q_ARG(int, cs.captureFrameRate),
+ Q_ARG(bool, preferCompressed),
+ Q_ARG(QString, cs.videoCodec), Q_ARG(int, cs.videoBitrateKbps * 1000),
+ Q_ARG(bool, vaapiEncoding));
QMetaObject::invokeMethod(cam, "Capture", Qt::QueuedConnection);
// remotevideo is configured lazily, once the peer's own 0x75 (see
diff --git a/pica-client/callsettings.cpp b/pica-client/callsettings.cpp
new file mode 100644
index 0000000..b97d46a
--- /dev/null
+++ b/pica-client/callsettings.cpp
@@ -0,0 +1,124 @@
+/*
+ (c) Copyright 2012 - 2026 Anton Sviridenko
+ https://picapica.im
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+#include "callsettings.h"
+#include "settings.h"
+#include "globals.h"
+
+const CallCodecInfo kCallVideoCodecs[] =
+{
+ { "h264", "H.264" },
+ { "hevc", "H.265/HEVC" },
+ { "vp9", "VP9" }
+};
+
+const int kCallVideoCodecCount = sizeof(kCallVideoCodecs) / sizeof(kCallVideoCodecs[0]);
+
+const CallCodecInfo kCallAudioCodecs[] =
+{
+ { "opus", "Opus" },
+ { "adpcm_g722", "G.722" },
+ { "pcm_alaw", "G.711 A-law" }
+};
+
+const int kCallAudioCodecCount = sizeof(kCallAudioCodecs) / sizeof(kCallAudioCodecs[0]);
+
+int callAudioSampleRate(const QString &codec)
+{
+ // G.722 is a 16kHz codec by definition - it subband-codes a 7kHz audio
+ // bandwidth and there is no other rate to run it at.
+ if (codec == QLatin1String("adpcm_g722"))
+ return 16000;
+
+ // A-law is the telephone network's codec and 8kHz is what it is for.
+ // Running it at anything else would only spend twice the bandwidth on a
+ // codec chosen for costing as little as possible.
+ if (codec == QLatin1String("pcm_alaw"))
+ return 8000;
+
+ // Opus, and anything unrecognised: 16kHz rather than 48kHz because it is
+ // what the echo canceller wants. AEC3 does its filtering in the 0-8kHz
+ // band whatever it is handed; at 48kHz it splits the signal into four
+ // bands, cancels in the lowest and gates the rest, which is more work for
+ // a result that is no better on speech.
+ //
+ // Costs nothing worth having for speech: Opus codes this as wideband, an
+ // 8kHz audio bandwidth, which is well past telephone quality and past
+ // where voice intelligibility stops improving.
+ //
+ // Note that this is the capture direction only. Playback follows whatever
+ // the peer announced in its own 0x74 message (see
+ // AudioVideoCallController::incoming_audio_params()), so a call with a
+ // peer that picked a different codec or rate keeps working.
+ return 16000;
+}
+
+bool callAudioBitrateAdjustable(const QString &codec)
+{
+ // Opus is the only one of the three with a rate control worth the name;
+ // the other two are fixed bits per sample.
+ return codec == QLatin1String("opus") || codec.isEmpty();
+}
+
+int callAudioFixedBitrateKbps(const QString &codec)
+{
+ // G.722 is 4 bits per sample at 16kHz, A-law 8 bits per sample at 8kHz -
+ // 64 kbit/s either way, which is no coincidence.
+ if (codec == QLatin1String("adpcm_g722"))
+ return 16000 * 4 / 1000;
+
+ if (codec == QLatin1String("pcm_alaw"))
+ return 8000 * 8 / 1000;
+
+ return kDefaultAudioBitrateKbps;
+}
+
+CallSettings CallSettings::load()
+{
+ Settings st(config_dbname);
+ CallSettings cs;
+
+ cs.videoCodec = st.loadValue("call.video_codec", QLatin1String(kCallVideoCodecs[0].name)).toString();
+ cs.audioCodec = st.loadValue("call.audio_codec", QLatin1String(kCallAudioCodecs[0].name)).toString();
+
+ cs.videoBitrateKbps = st.loadValue("call.video_bitrate_kbps", kDefaultVideoBitrateKbps).toInt();
+ cs.audioBitrateKbps = st.loadValue("call.audio_bitrate_kbps", kDefaultAudioBitrateKbps).toInt();
+
+ cs.captureWidth = st.loadValue("video.capture_width", kDefaultCaptureWidth).toInt();
+ cs.captureHeight = st.loadValue("video.capture_height", kDefaultCaptureHeight).toInt();
+ cs.captureFrameRate = st.loadValue("video.capture_framerate", kDefaultCaptureFrameRate).toInt();
+
+ // A stored value that makes no sense - a hand-edited database, or a
+ // setting written by a build that offered something this one does not -
+ // would otherwise be passed to the encoder and fail the call rather than
+ // the setting.
+ if (cs.videoBitrateKbps <= 0)
+ cs.videoBitrateKbps = kDefaultVideoBitrateKbps;
+
+ if (cs.audioBitrateKbps <= 0)
+ cs.audioBitrateKbps = kDefaultAudioBitrateKbps;
+
+ if (cs.captureWidth <= 0 || cs.captureHeight <= 0)
+ {
+ cs.captureWidth = kDefaultCaptureWidth;
+ cs.captureHeight = kDefaultCaptureHeight;
+ }
+
+ if (cs.captureFrameRate <= 0)
+ cs.captureFrameRate = kDefaultCaptureFrameRate;
+
+ return cs;
+}
diff --git a/pica-client/callsettings.h b/pica-client/callsettings.h
new file mode 100644
index 0000000..65419e4
--- /dev/null
+++ b/pica-client/callsettings.h
@@ -0,0 +1,106 @@
+/*
+ (c) Copyright 2012 - 2026 Anton Sviridenko
+ https://picapica.im
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+#ifndef CALLSETTINGS_H
+#define CALLSETTINGS_H
+
+#include <QString>
+
+// The parameters an audio/video call is started with: what the settings
+// dialog's "Call Settings" and "Video Devices" tabs write, and what
+// AudioVideoCallController reads when a call begins. Both sides go through
+// this one place so they cannot drift apart, and so that "never configured"
+// means the same thing to each of them.
+//
+// Codec names are FFmpeg names throughout. That is also how the 0x74 and 0x75
+// protocol messages name them (see doc/proto-doc-latest.txt), so what is
+// stored here goes both onto the wire and into
+// avcodec_find_encoder_by_name() with no translation table in between.
+
+// One entry of the codec lists the settings dialog offers.
+struct CallCodecInfo
+{
+ // The FFmpeg name, which is what gets stored and what goes on the wire.
+ const char *name;
+ // What the settings dialog shows for it.
+ const char *displayName;
+};
+
+// Video codecs offered by the "Video Codec" setting, in the order they are
+// listed. Every one of them is a codec the 0x75 message can name and that the
+// receiving side hands to avcodec_find_decoder_by_name(), so no peer needs to
+// know in advance which one we picked.
+extern const CallCodecInfo kCallVideoCodecs[];
+extern const int kCallVideoCodecCount;
+
+// Audio codecs offered by the "Audio Codec" setting - the three the protocol
+// names for the 0x74 message.
+extern const CallCodecInfo kCallAudioCodecs[];
+extern const int kCallAudioCodecCount;
+
+// What a call uses when nothing has ever been configured: the values that were
+// compiled in before any of this became a setting, so an install that has
+// never opened the settings dialog behaves exactly as it did.
+const int kDefaultVideoBitrateKbps = 600;
+const int kDefaultAudioBitrateKbps = 32;
+const int kDefaultCaptureWidth = 640;
+const int kDefaultCaptureHeight = 480;
+const int kDefaultCaptureFrameRate = 15;
+
+struct CallSettings
+{
+ // FFmpeg codec names, e.g. "h264" and "opus".
+ QString videoCodec;
+ QString audioCodec;
+
+ // As the user enters them, in kbit/s.
+ int videoBitrateKbps;
+ int audioBitrateKbps;
+
+ // What the camera is asked for. Not necessarily what it delivers - see
+ // VideoDevice::Capture(), which loosens these a step at a time rather than
+ // failing when a camera will not honour them.
+ int captureWidth;
+ int captureHeight;
+ int captureFrameRate;
+
+ // Reads the stored settings, filling in the defaults above for anything
+ // that was never set. Must be called from a thread that may touch the
+ // settings database - not from a capture or playback loop, see
+ // AudioDevice::SetLinuxDriverName() for why.
+ static CallSettings load();
+};
+
+// The rate a given audio codec is captured and encoded at, and announced at in
+// the 0x74 message.
+//
+// It is a property of the codec rather than a setting: G.722 is defined at
+// 16kHz and nowhere else, and A-law is a telephony codec whose whole point is
+// 8kHz. Opus would take any of them and uses 16kHz for the reason spelled out
+// in callsettings.cpp.
+int callAudioSampleRate(const QString &codec);
+
+// Whether the codec's bitrate is the caller's to choose. Opus is; G.722 and
+// A-law each pack a fixed number of bits per sample, so their bitrate follows
+// from the sample rate and nothing can be asked of the encoder.
+bool callAudioBitrateAdjustable(const QString &codec);
+
+// The bitrate a fixed rate codec ends up at, in kbit/s, for showing in the
+// settings dialog. Meaningless for a codec callAudioBitrateAdjustable() is
+// true for, which returns the configured value instead.
+int callAudioFixedBitrateKbps(const QString &codec);
+
+#endif // CALLSETTINGS_H
diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp
index 8685936..15c7c96 100644
--- a/pica-client/dialogs/settingsdialog.cpp
+++ b/pica-client/dialogs/settingsdialog.cpp
@@ -19,25 +19,28 @@
#include "../globals.h"
#include "../audiodevice.h"
#include "../videodevice.h"
+#include "../callsettings.h"
#include "../../PICA_netconf.h"
#include "../../PICA_proto.h"
#include <QGroupBox>
#include <QVBoxLayout>
#include <QHBoxLayout>
+#include <QFormLayout>
#include <QLabel>
#include <QDebug>
#include <QVariant>
#include <QListWidget>
#include <QNetworkInterface>
#include <QHostAddress>
+#include <QSize>
#include <QTabWidget>
#include <QTimer>
-// Audio pipeline test: how long the microphone is recorded for, and the
-// sample rate to record at - the same rate a call negotiates.
+// Audio pipeline test: how long the microphone is recorded for. The codec and
+// the rate come from the "Call Settings" tab, so the test runs the same path a
+// call would - see toggleAudioTest().
static const int kAudioTestRecordMs = 5000;
-static const int kAudioTestSampleRate = 48000;
// One encoded packet covers this much audio (the Opus encoder produces
// 20 ms frames at 48 kHz), so feeding one packet per this many milliseconds
@@ -55,14 +58,41 @@ static const int kAudioTestStartTimeoutMs = 4000;
// buffer and the sound card's own buffer to drain before closing the device.
static const int kAudioTestDrainMs = 500;
-// Size requested from the camera by the video pipeline test, matching what a
-// call asks for.
-static const int kVideoTestWidth = 640;
-static const int kVideoTestHeight = 480;
+// Sizes and frame rates offered when the camera will not say what it supports
+// - a platform where they cannot be queried, or a camera that will not answer.
+// The camera is free to refuse any of them, the same way it may refuse
+// anything else it was asked for; VideoDevice::Capture() loosens the request
+// rather than failing.
+static const struct
+{
+ int width;
+ int height;
+} kFallbackResolutions[] =
+{
+ { 1920, 1080 }, { 1280, 720 }, { 800, 600 }, { 640, 480 },
+ { 640, 360 }, { 352, 288 }, { 320, 240 }, { 160, 120 }
+};
+
+static const int kFallbackFrameRates[] = { 30, 25, 24, 20, 15, 10, 5 };
+
+// Range the bitrate spin boxes accept, in kbit/s. Wide enough not to argue
+// with anyone: the low end is where a codec still produces something, the high
+// end well past what a call over a home connection can carry.
+static const int kMinVideoBitrateKbps = 50;
+static const int kMaxVideoBitrateKbps = 20000;
+static const int kMinAudioBitrateKbps = 6;
+static const int kMaxAudioBitrateKbps = 510;
SettingsDialog::SettingsDialog(QWidget *parent) :
QDialog(parent)
{
+ // The video tab's format lists follow the camera selection, and
+ // fillVideoDevices() below changes that before they have been built - so
+ // they have to be readable as "not there yet" from the moment anything can
+ // reach them.
+ videoRes = nullptr;
+ videoFps = nullptr;
+
QVBoxLayout *settingsLayout = new QVBoxLayout();
QTabWidget *tabW = new QTabWidget(this);
@@ -71,11 +101,13 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
QWidget* multilogintab = new QWidget(0);
QWidget* audiodevtab = new QWidget(0);
QWidget* videodevtab = new QWidget(0);
+ QWidget* calltab = new QWidget(0);
QVBoxLayout *directc2cLayout = new QVBoxLayout();
QVBoxLayout *multiloginlayout = new QVBoxLayout();
QVBoxLayout *audiodevlayout = new QVBoxLayout();
QVBoxLayout *videodevLayout = new QVBoxLayout();
+ QVBoxLayout *callLayout = new QVBoxLayout();
//Direct connections
rbDisableDirectConns = new QRadioButton(tr("Disable direct connections"), this);
@@ -305,6 +337,29 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
videoDevRefresh = new QPushButton(tr("Refresh 🔄"), this);
connect(videoDevRefresh, SIGNAL(clicked()), this, SLOT(fillVideoDevices()));
+ QLabel *lbVideoRes = new QLabel(tr("Resolution"), this);
+ videoRes = new QComboBox(this);
+ videoRes->setToolTip(tr("Picture sizes the selected camera offers. A size the camera can "
+ "deliver already compressed lists the formats in brackets; those "
+ "are the ones the \"prefer compressed formats\" setting below can "
+ "forward untouched, without decoding and re-encoding them here."));
+
+ QLabel *lbVideoFps = new QLabel(tr("Frame rate"), this);
+ videoFps = new QComboBox(this);
+ videoFps->setToolTip(tr("Frame rates the camera offers at the selected size."));
+
+ // Filled from the camera the device list above has selected, so the two
+ // have to follow it - and each other.
+ fillVideoResolutions();
+ connect(videoDev, SIGNAL(currentIndexChanged(int)), this, SLOT(fillVideoResolutions()));
+ connect(videoRes, SIGNAL(currentIndexChanged(int)), this, SLOT(fillVideoFrameRates()));
+
+ QHBoxLayout *videoFormatLayout = new QHBoxLayout();
+ videoFormatLayout->addWidget(lbVideoRes);
+ videoFormatLayout->addWidget(videoRes, 1);
+ videoFormatLayout->addWidget(lbVideoFps);
+ videoFormatLayout->addWidget(videoFps);
+
cbPreferCompressed = new QCheckBox(tr("Prefer compressed formats if provided by the camera"), this);
#ifdef HAVE_VAAPI
@@ -336,6 +391,7 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
videodevLayout->addWidget(videoDev);
videodevLayout->addWidget(videoDevRefresh);
+ videodevLayout->addLayout(videoFormatLayout);
videodevLayout->addWidget(cbPreferCompressed);
#ifdef HAVE_VAAPI
videodevLayout->addWidget(cbVaapiEncoding);
@@ -376,10 +432,73 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
#endif
testDecoderThread.start();
+//Call Settings
+ // What a call encodes with. The codec names stored here are FFmpeg names,
+ // which is also how the 0x74 and 0x75 messages name them - see
+ // callsettings.h.
+ callVideoCodec = new QComboBox(this);
+
+ for (int i = 0; i < kCallVideoCodecCount; i++)
+ callVideoCodec->addItem(QLatin1String(kCallVideoCodecs[i].displayName),
+ QLatin1String(kCallVideoCodecs[i].name));
+
+ callVideoCodec->setToolTip(tr("What outgoing video is encoded with. The peer is told which "
+ "codec is in use, so it does not have to be one it was "
+ "configured for.\n\n"
+ "This applies to video encoded here. With \"prefer compressed "
+ "formats\" switched on and a camera that delivers a compressed "
+ "stream itself, that stream is forwarded as it comes and the "
+ "camera's format is what the peer receives - this setting then "
+ "only decides which of the camera's formats is preferred."));
+
+ callAudioCodec = new QComboBox(this);
+
+ for (int i = 0; i < kCallAudioCodecCount; i++)
+ callAudioCodec->addItem(QLatin1String(kCallAudioCodecs[i].displayName),
+ QLatin1String(kCallAudioCodecs[i].name));
+
+ callAudioCodec->setToolTip(tr("What outgoing audio is encoded with. Opus is the one to want "
+ "for a voice call; G.722 and G.711 A-law are the telephony "
+ "codecs, fixed at 64 kbit/s and there for compatibility."));
+
+ callVideoBitrate = new QSpinBox(this);
+ callVideoBitrate->setRange(kMinVideoBitrateKbps, kMaxVideoBitrateKbps);
+ callVideoBitrate->setSuffix(tr(" kbps"));
+ callVideoBitrate->setValue(kDefaultVideoBitrateKbps);
+
+ callAudioBitrate = new QSpinBox(this);
+ callAudioBitrate->setRange(kMinAudioBitrateKbps, kMaxAudioBitrateKbps);
+ callAudioBitrate->setSuffix(tr(" kbps"));
+ callAudioBitrate->setValue(kDefaultAudioBitrateKbps);
+ audioBitrateKbps = kDefaultAudioBitrateKbps;
+
+ connect(callAudioCodec, SIGNAL(currentIndexChanged(int)), this, SLOT(callAudioCodecChanged()));
+
+ QFormLayout *callFormLayout = new QFormLayout();
+ callFormLayout->addRow(tr("Video Codec"), callVideoCodec);
+ callFormLayout->addRow(tr("Video Bitrate"), callVideoBitrate);
+ callFormLayout->addRow(tr("Audio Codec"), callAudioCodec);
+ callFormLayout->addRow(tr("Audio Bitrate"), callAudioBitrate);
+
+ // The other half of what a call is started with lives on the video tab,
+ // next to the camera that has to deliver it; say so rather than leaving
+ // this tab looking like the whole story.
+ QLabel *lbCallHint = new QLabel(tr("The picture size and frame rate a call is started with are "
+ "on the Video Devices tab, next to the camera that has to "
+ "deliver them."), this);
+ lbCallHint->setWordWrap(true);
+
+ callLayout->addLayout(callFormLayout);
+ callLayout->addWidget(lbCallHint);
+ callLayout->addStretch(1);
+
+ calltab->setLayout(callLayout);
+
tabW->addTab(directc2ctab, tr("Direct Connections"));
tabW->addTab(multilogintab, tr("Multiple logins"));
tabW->addTab(audiodevtab, tr("Audio Devices"));
tabW->addTab(videodevtab, tr("Video Devices"));
+ tabW->addTab(calltab, tr("Call Settings"));
tabW->addTab(soundstab, tr("Sounds"));
settingsLayout->addWidget(tabW);
@@ -511,9 +630,16 @@ void SettingsDialog::toggleAudioTest()
btAudioTest->setText(tr("Stop ⏹"));
audioTestStatus->setText(tr("Recording, say something..."));
+ // The codec selected on the "Call Settings" tab, at the rate a call with
+ // it would run at - same as the video test, this exercises what the
+ // current selections do rather than a fixed pipeline of its own.
+ const QString codec = callAudioCodec->itemData(callAudioCodec->currentIndex()).toString();
+ const int rate = callAudioSampleRate(codec);
+ const int bitrate = callAudioBitrateAdjustable(codec) ? callAudioBitrate->value() * 1000 : 0;
+
QMetaObject::invokeMethod(testMic, "configureCapture", Qt::QueuedConnection,
- Q_ARG(QString, dev), Q_ARG(QString, QStringLiteral("opus")),
- Q_ARG(int, kAudioTestSampleRate));
+ Q_ARG(QString, dev), Q_ARG(QString, codec),
+ Q_ARG(int, rate), Q_ARG(int, bitrate));
QMetaObject::invokeMethod(testMic, "Capture", Qt::QueuedConnection);
// Only a watchdog for the device failing to open at all. The recording
@@ -589,9 +715,13 @@ void SettingsDialog::startAudioTestPlayback()
audioTestStatus->setText(summary);
+ // Decoded with what it was encoded with, exactly as the receiving end of a
+ // call decodes with whatever the peer announced.
+ const QString codec = callAudioCodec->itemData(callAudioCodec->currentIndex()).toString();
+
QMetaObject::invokeMethod(testSpeaker, "configurePlayback", Qt::QueuedConnection,
- Q_ARG(QString, dev), Q_ARG(QString, QStringLiteral("opus")),
- Q_ARG(int, kAudioTestSampleRate));
+ Q_ARG(QString, dev), Q_ARG(QString, codec),
+ Q_ARG(int, callAudioSampleRate(codec)));
QMetaObject::invokeMethod(testSpeaker, "Play", Qt::QueuedConnection);
// The recording has to be handed over at the rate it plays at, not all at
@@ -715,10 +845,16 @@ void SettingsDialog::toggleVideoTest()
videoTestPathReport.clear();
+ // The settings as they stand in the dialog, not as they were last stored:
+ // the point of the test is to see what the current selections do before
+ // committing to them.
QMetaObject::invokeMethod(testCam, "configureCapture", Qt::QueuedConnection,
Q_ARG(QString, dev),
- Q_ARG(int, kVideoTestWidth), Q_ARG(int, kVideoTestHeight),
+ Q_ARG(int, selectedVideoWidth()), Q_ARG(int, selectedVideoHeight()),
+ Q_ARG(int, selectedVideoFrameRate()),
Q_ARG(bool, cbPreferCompressed->isChecked()),
+ Q_ARG(QString, callVideoCodec->itemData(callVideoCodec->currentIndex()).toString()),
+ Q_ARG(int, callVideoBitrate->value() * 1000),
Q_ARG(bool, vaapiEncoding));
QMetaObject::invokeMethod(testCam, "Capture", Qt::QueuedConnection);
@@ -898,10 +1034,11 @@ void SettingsDialog::fillDevicesComboBox(QComboBox *cb, MediaDevice *dev, enum M
.arg(md.at(i).device)
.arg(md.at(i).humanReadable);
- // Cameras that can deliver a compressed stream themselves say so,
- // which is what the "prefer compressed formats" setting acts on.
- if (!md.at(i).compressedFormats.isEmpty())
- item += QString(QLatin1String(" [%1]")).arg(md.at(i).compressedFormats.join(QLatin1String(", ")));
+ // A camera's compressed formats are not listed here but in the
+ // resolution list, which is where they actually mean something: a
+ // camera offers them at some of its sizes and not at others, so
+ // naming them against the camera as a whole said less than it
+ // appeared to.
cb->addItem(item, md.at(i).device);
}
@@ -910,7 +1047,193 @@ void SettingsDialog::fillDevicesComboBox(QComboBox *cb, MediaDevice *dev, enum M
void SettingsDialog::fillVideoDevices()
{
VideoDevice vd;
+
+ // Refilling walks the current index through -1 and back, and each step
+ // of that would otherwise ask a camera what it can do - which is not
+ // free, on Windows it means instantiating the capture filter. So the
+ // format lists below are rebuilt once, at the end.
+ const QString previous = videoDev->itemData(videoDev->currentIndex()).toString();
+
+ videoDev->blockSignals(true);
fillDevicesComboBox(videoDev, &vd, CAPTURE);
+
+ // Refresh is for picking up a camera that was plugged in, not for
+ // moving the selection off the one already chosen.
+ int item = videoDev->findData(previous);
+ if (item >= 0)
+ videoDev->setCurrentIndex(item);
+
+ videoDev->blockSignals(false);
+
+ fillVideoResolutions();
+}
+
+void SettingsDialog::fillVideoResolutions()
+{
+ // Nothing to fill into yet while the constructor is still building the
+ // tab - fillVideoDevices() runs before these exist.
+ if (!videoRes || !videoFps)
+ return;
+
+ const QString dev = videoDev->itemData(videoDev->currentIndex()).toString();
+
+ videoFormats = dev.isEmpty() ? QList<VideoCaptureFormat>()
+ : VideoDevice::CaptureFormats(dev);
+
+ // Refilling moves the current index about; the frame rate list is rebuilt
+ // once at the end rather than on every one of those intermediate changes.
+ videoRes->blockSignals(true);
+ videoRes->clear();
+
+ if (videoFormats.isEmpty())
+ {
+ // Either the camera would not say or this platform cannot ask. Offer
+ // the common sizes rather than an empty list - a camera that cannot
+ // produce one of them is handled the same way as any other request it
+ // will not honour, see VideoDevice::Capture().
+ for (unsigned int i = 0; i < sizeof(kFallbackResolutions) / sizeof(kFallbackResolutions[0]); i++)
+ {
+ const int w = kFallbackResolutions[i].width;
+ const int h = kFallbackResolutions[i].height;
+
+ videoRes->addItem(QString(QLatin1String("%1x%2")).arg(w).arg(h), QSize(w, h));
+ }
+ }
+ else
+ {
+ for (int i = 0; i < videoFormats.size(); i++)
+ {
+ const VideoCaptureFormat &f = videoFormats.at(i);
+ QString item = QString(QLatin1String("%1x%2")).arg(f.width).arg(f.height);
+
+ // The formats this camera can hand over ready-made at this size,
+ // which is what "prefer compressed formats" below acts on.
+ if (!f.compressedFormats.isEmpty())
+ item += QString(QLatin1String(" [%1]")).arg(f.compressedFormats.join(QLatin1String(", ")));
+
+ videoRes->addItem(item, QSize(f.width, f.height));
+ }
+ }
+
+ // Keep the configured size selected across a device change where the new
+ // camera offers it too, rather than silently moving the setting to
+ // whatever happens to be first in the new list.
+ Settings st(config_dbname);
+ QSize wanted(st.loadValue("video.capture_width", kDefaultCaptureWidth).toInt(),
+ st.loadValue("video.capture_height", kDefaultCaptureHeight).toInt());
+
+ int item = videoRes->findData(wanted);
+
+ if (item < 0)
+ item = videoRes->findData(QSize(kDefaultCaptureWidth, kDefaultCaptureHeight));
+
+ if (item >= 0)
+ videoRes->setCurrentIndex(item);
+
+ videoRes->blockSignals(false);
+
+ fillVideoFrameRates();
+}
+
+void SettingsDialog::fillVideoFrameRates()
+{
+ if (!videoFps)
+ return;
+
+ const QSize size = videoRes->itemData(videoRes->currentIndex()).toSize();
+ QList<int> rates;
+
+ for (int i = 0; i < videoFormats.size(); i++)
+ {
+ if (videoFormats.at(i).width == size.width() && videoFormats.at(i).height == size.height())
+ {
+ rates = videoFormats.at(i).frameRates;
+ break;
+ }
+ }
+
+ videoFps->blockSignals(true);
+ videoFps->clear();
+
+ if (rates.isEmpty())
+ {
+ // Same reasoning as for the sizes: a camera that named its sizes but
+ // not its rates, or one we could not ask at all.
+ for (unsigned int i = 0; i < sizeof(kFallbackFrameRates) / sizeof(kFallbackFrameRates[0]); i++)
+ rates << kFallbackFrameRates[i];
+ }
+
+ for (int i = 0; i < rates.size(); i++)
+ videoFps->addItem(tr("%1 fps").arg(rates.at(i)), rates.at(i));
+
+ Settings st(config_dbname);
+ int wanted = st.loadValue("video.capture_framerate", kDefaultCaptureFrameRate).toInt();
+
+ int item = videoFps->findData(wanted);
+
+ if (item < 0)
+ item = videoFps->findData(kDefaultCaptureFrameRate);
+
+ // Neither the configured rate nor the default is on offer at this size -
+ // take the highest the camera does offer, which is the first in the list.
+ if (item < 0 && videoFps->count() > 0)
+ item = 0;
+
+ if (item >= 0)
+ videoFps->setCurrentIndex(item);
+
+ videoFps->blockSignals(false);
+}
+
+int SettingsDialog::selectedVideoWidth() const
+{
+ const QSize size = videoRes->itemData(videoRes->currentIndex()).toSize();
+
+ return size.isValid() ? size.width() : kDefaultCaptureWidth;
+}
+
+int SettingsDialog::selectedVideoHeight() const
+{
+ const QSize size = videoRes->itemData(videoRes->currentIndex()).toSize();
+
+ return size.isValid() ? size.height() : kDefaultCaptureHeight;
+}
+
+int SettingsDialog::selectedVideoFrameRate() const
+{
+ bool ok = false;
+ const int fps = videoFps->itemData(videoFps->currentIndex()).toInt(&ok);
+
+ return (ok && fps > 0) ? fps : kDefaultCaptureFrameRate;
+}
+
+void SettingsDialog::callAudioCodecChanged()
+{
+ const QString codec = callAudioCodec->itemData(callAudioCodec->currentIndex()).toString();
+
+ if (callAudioBitrateAdjustable(codec))
+ {
+ // Coming back from a fixed rate codec, whose rate is showing in the
+ // box; the user's own choice is in audioBitrateKbps.
+ if (!callAudioBitrate->isEnabled())
+ {
+ callAudioBitrate->setEnabled(true);
+ callAudioBitrate->setValue(audioBitrateKbps);
+ callAudioBitrate->setToolTip(QString());
+ }
+
+ return;
+ }
+
+ if (callAudioBitrate->isEnabled())
+ audioBitrateKbps = callAudioBitrate->value();
+
+ // Show what the codec will actually run at rather than leaving a number
+ // behind that has nothing to do with it.
+ callAudioBitrate->setEnabled(false);
+ callAudioBitrate->setValue(callAudioFixedBitrateKbps(codec));
+ callAudioBitrate->setToolTip(tr("This codec codes a fixed number of bits per sample, so its "
+ "bitrate follows from the sample rate and cannot be chosen."));
}
void SettingsDialog::audioDriverChanged(int index)
@@ -1099,8 +1422,38 @@ void SettingsDialog::loadSettings()
if (videoDevItem >= 0)
videoDev->setCurrentIndex(videoDevItem);
+ // The resolution and frame rate lists pick the stored size and rate out of
+ // what the selected camera offers as they are filled, which the line above
+ // has just triggered where it changed the camera - see
+ // fillVideoResolutions().
+
cbPreferCompressed->setChecked(st.loadValue("video.prefer_compressed", 0).toBool());
+ // Call encoding parameters. Read through CallSettings rather than from st
+ // directly, so that the dialog and a call that is started without ever
+ // opening it agree on what "not configured" means.
+ const CallSettings cs = CallSettings::load();
+
+ int callVideoCodecItem = callVideoCodec->findData(cs.videoCodec);
+ if (callVideoCodecItem >= 0)
+ callVideoCodec->setCurrentIndex(callVideoCodecItem);
+
+ callVideoBitrate->setValue(qBound(kMinVideoBitrateKbps, cs.videoBitrateKbps, kMaxVideoBitrateKbps));
+
+ // Before the codec is selected: selecting one whose bitrate is fixed puts
+ // that codec's rate in the box and keeps this as the value to come back
+ // to, see callAudioCodecChanged().
+ audioBitrateKbps = qBound(kMinAudioBitrateKbps, cs.audioBitrateKbps, kMaxAudioBitrateKbps);
+ callAudioBitrate->setValue(audioBitrateKbps);
+
+ int callAudioCodecItem = callAudioCodec->findData(cs.audioCodec);
+ if (callAudioCodecItem >= 0)
+ callAudioCodec->setCurrentIndex(callAudioCodecItem);
+
+ // Explicitly, because setCurrentIndex() above only emits when the index
+ // actually changes - and the stored codec is often the one already showing.
+ callAudioCodecChanged();
+
#ifdef HAVE_VAAPI
cbVaapiEncoding->setChecked(st.loadValue("video.vaapi_encoding", 0).toBool());
cbVaapiDecoding->setChecked(st.loadValue("video.vaapi_decoding", 0).toBool());
@@ -1182,7 +1535,22 @@ void SettingsDialog::storeSettings()
st.storeValue("multiple_logins.state", QString::number(mlpstate));
st.storeValue("video.capture_device", videoDev->itemData(videoDev->currentIndex()).toString());
+ st.storeValue("video.capture_width", QString::number(selectedVideoWidth()));
+ st.storeValue("video.capture_height", QString::number(selectedVideoHeight()));
+ st.storeValue("video.capture_framerate", QString::number(selectedVideoFrameRate()));
st.storeValue("video.prefer_compressed", cbPreferCompressed->isChecked() ? "1" : "0");
+
+ st.storeValue("call.video_codec", callVideoCodec->itemData(callVideoCodec->currentIndex()).toString());
+ st.storeValue("call.audio_codec", callAudioCodec->itemData(callAudioCodec->currentIndex()).toString());
+ st.storeValue("call.video_bitrate_kbps", QString::number(callVideoBitrate->value()));
+
+ // The box shows a fixed rate codec's own bitrate while such a codec is
+ // selected; what gets stored is the user's choice either way, so that
+ // switching back to Opus finds it again on the next visit.
+ if (callAudioBitrate->isEnabled())
+ audioBitrateKbps = callAudioBitrate->value();
+
+ st.storeValue("call.audio_bitrate_kbps", QString::number(audioBitrateKbps));
#ifdef HAVE_VAAPI
st.storeValue("video.vaapi_encoding", cbVaapiEncoding->isChecked() ? "1" : "0");
st.storeValue("video.vaapi_decoding", cbVaapiDecoding->isChecked() ? "1" : "0");
diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h
index ba04612..2167f5f 100644
--- a/pica-client/dialogs/settingsdialog.h
+++ b/pica-client/dialogs/settingsdialog.h
@@ -94,6 +94,32 @@ private:
QComboBox *videoDev;
QPushButton *videoDevRefresh;
+ // Picture size and frame rate to ask the selected camera for. Both are
+ // filled from what the camera says it can do - the sizes from the camera,
+ // the rates from the size selected next to them - so they are refilled
+ // whenever the selection above them changes.
+ QComboBox *videoRes;
+ QComboBox *videoFps;
+
+ // What the selected camera reported, kept so that changing the resolution
+ // can refill the frame rates without asking the camera again. Enumerating
+ // is not free - on Windows it means instantiating the capture filter.
+ QList<VideoCaptureFormat> videoFormats;
+
+ // Encoding parameters for a call, on the "Call Settings" tab. The codec
+ // combo boxes carry the FFmpeg codec name as item data, which is what gets
+ // stored and what goes into the 0x74/0x75 messages.
+ QComboBox *callVideoCodec;
+ QComboBox *callAudioCodec;
+ QSpinBox *callVideoBitrate;
+ QSpinBox *callAudioBitrate;
+
+ // The audio bitrate the user chose, which is not always what
+ // callAudioBitrate shows: selecting a fixed rate codec puts that codec's
+ // rate in the box and disables it, and the chosen value has to survive
+ // that to come back when an adjustable codec is selected again.
+ int audioBitrateKbps;
+
// Local audio pipeline test: a few seconds of the selected microphone are
// captured and encoded, then decoded again and played back through the
// selected playback device - the same path a call takes, minus the
@@ -184,12 +210,25 @@ private:
void fillDevicesComboBox(QComboBox *cb, MediaDevice *dev, enum MediaDeviceStreamDirection dir);
+ // The picture size and frame rate the video tab currently has selected,
+ // used both by the local pipeline test and when the settings are stored.
+ int selectedVideoWidth() const;
+ int selectedVideoHeight() const;
+ int selectedVideoFrameRate() const;
+
private slots:
void OK();
void Cancel();
void toggleIncomingConnections(bool checked);
void toggleMultipleLogins(bool checked);
void fillVideoDevices();
+ // Asks the newly selected camera what it can do and refills the two lists
+ // below it. fillVideoFrameRates() alone when only the size changed.
+ void fillVideoResolutions();
+ void fillVideoFrameRates();
+ // Greys out the audio bitrate box for a codec whose bitrate is not ours to
+ // choose - see audioBitrateKbps.
+ void callAudioCodecChanged();
// Device names are driver specific, so all three lists have to be filled
// again when the driver changes. Only ever connected on Linux.
void audioDriverChanged(int index);
diff --git a/pica-client/mediadevice.h b/pica-client/mediadevice.h
index 542674b..932c8e8 100644
--- a/pica-client/mediadevice.h
+++ b/pica-client/mediadevice.h
@@ -16,10 +16,6 @@ struct MediaDeviceInfo
QString device;
QString humanReadable;
int index;
- // Compressed formats this device can deliver by itself, as FFmpeg codec
- // names, most preferred first. Filled in for cameras that offer such a
- // stream (see VideoDevice::CompressedFormats), empty otherwise.
- QStringList compressedFormats;
};
class MediaDevice
diff --git a/pica-client/pica-client.pro b/pica-client/pica-client.pro
index 279879a..4823c8a 100644
--- a/pica-client/pica-client.pro
+++ b/pica-client/pica-client.pro
@@ -48,6 +48,7 @@ SOURCES += main.cpp\
sound.cpp \
dialogs/settingsdialog.cpp \
settings.cpp \
+ callsettings.cpp \
../PICA_netconf.c \
../PICA_signverify.c \
dialogs/nodesdialog.cpp \
@@ -93,6 +94,7 @@ HEADERS += mainwindow.h \
sound.h \
dialogs/settingsdialog.h \
settings.h \
+ callsettings.h \
../PICA_netconf.h \
../PICA_security.h \
../PICA_signverify.h \
diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp
index 5b7d5fb..3fb468c 100644
--- a/pica-client/videodevice.cpp
+++ b/pica-client/videodevice.cpp
@@ -15,13 +15,18 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "videodevice.h"
+#include "callsettings.h"
#include "../PICA_proto.h"
#include "../PICA_media.h"
#include <QFile>
#include <QDebug>
#include <QMutexLocker>
+#include <QMap>
+#include <QPair>
#include <cstring>
+#include <algorithm>
+#include <functional>
#ifdef Q_OS_LINUX
#include <linux/videodev2.h>
@@ -54,11 +59,6 @@ extern "C" {
#endif
}
-// Encoding defaults. Not exposed as settings yet - see the deferred list in
-// the feature plan; a fixed, widely supported set of parameters for now.
-static const int kFrameRate = 15;
-static const int64_t kBitrate = 600000;
-
// Largest amount of encoded data one 0x77 message can carry over a c2c or
// directc2c connection: the protocol's per-message payload cap, minus the
// sequence number + timestamp header that every call media packet carries.
@@ -106,6 +106,131 @@ static const struct
};
#endif
+// Which FFmpeg encoder produces each of the codecs the "Video Codec" setting
+// offers, on the GPU and in software. Two names per codec because FFmpeg names
+// its external library encoders after the library rather than after the codec,
+// and the id as well so that a build without the library named here can still
+// fall back to whatever other encoder it has for the same codec.
+static const struct
+{
+ const char *codec;
+ AVCodecID id;
+ const char *vaapiEncoder;
+ const char *swEncoder;
+} kVideoEncoders[] =
+{
+ { "h264", AV_CODEC_ID_H264, "h264_vaapi", "libx264" },
+ { "hevc", AV_CODEC_ID_HEVC, "hevc_vaapi", "libx265" },
+ { "vp9", AV_CODEC_ID_VP9, "vp9_vaapi", "libvpx-vp9" }
+};
+
+static int videoEncoderIndex(const QString &codec)
+{
+ for (unsigned int i = 0; i < sizeof(kVideoEncoders) / sizeof(kVideoEncoders[0]); i++)
+ {
+ if (codec == QLatin1String(kVideoEncoders[i].codec))
+ return (int)i;
+ }
+
+ // An unrecognised setting - written by a build offering something this one
+ // does not, or by hand - falls back to the codec every peer can decode
+ // rather than leaving the call without video.
+ return 0;
+}
+
+// Sets the private options that make an encoder suitable for a live call:
+// low latency, no frame reordering, and parameter sets the receiver can pick
+// up mid-stream. Each of the three encoders spells all of that differently,
+// which is why this is keyed on the encoder rather than on the codec.
+static void applyLiveEncoderOptions(AVCodecContext *enc_ctx, const QString &encoderName, int sliceMaxSize)
+{
+ if (encoderName == QLatin1String("libx264"))
+ {
+ av_opt_set(enc_ctx->priv_data, "preset", "veryfast", 0);
+ av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0);
+
+ // Keeps every slice within one datagram of a mediac2c connection, so
+ // that a lost fragment costs a slice of the picture rather than the
+ // whole frame. Harmless when the media goes over TCP instead - it
+ // only splits frames into more NAL units than strictly necessary.
+ av_opt_set(enc_ctx->priv_data, "x264opts",
+ QString("slice-max-size=%1").arg(sliceMaxSize).toUtf8().constData(), 0);
+ }
+ else if (encoderName == QLatin1String("libx265"))
+ {
+ av_opt_set(enc_ctx->priv_data, "preset", "ultrafast", 0);
+ av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0);
+
+ // repeat-headers is the x265 equivalent of leaving
+ // AV_CODEC_FLAG_GLOBAL_HEADER off below: without it the VPS/SPS/PPS go
+ // out once, at the start, and a receiver that joins the stream later -
+ // which every receiver does, the call being already running by the
+ // time its decoder opens - never sees them.
+ //
+ // x265 has no slice-max-size; the nearest thing is a fixed number of
+ // slices per frame, which does not bound their size. Fragmentation
+ // handles oversized frames either way, a loss just costs more of the
+ // picture than it does with x264.
+ av_opt_set(enc_ctx->priv_data, "x265-params", "repeat-headers=1:bframes=0", 0);
+ }
+ else if (encoderName == QLatin1String("libvpx-vp9"))
+ {
+ // "realtime" with a high cpu-used is libvpx's zerolatency: it caps how
+ // long the encoder may spend per frame rather than letting it take as
+ // long as the quality target needs.
+ av_opt_set(enc_ctx->priv_data, "deadline", "realtime", 0);
+ av_opt_set_int(enc_ctx->priv_data, "cpu-used", 8, 0);
+ // No lookahead - it would hold frames back, which is latency in a
+ // live call - and keep the bitstream decodable across a loss.
+ av_opt_set_int(enc_ctx->priv_data, "lag-in-frames", 0, 0);
+ av_opt_set_int(enc_ctx->priv_data, "error-resilient", 1, 0);
+
+ // VP9 carries no parameter sets to repeat: everything a decoder needs
+ // is in the keyframe itself, so joining mid-stream needs nothing
+ // special here.
+ }
+}
+
+#ifdef Q_OS_WIN
+// Defined in videodshow.cpp, alongside the enumeration functions - see the
+// comment on pica_enumerate_dshow_video() for why the answers come from
+// DirectShow rather than from FFmpeg's demuxer.
+QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device);
+#endif
+
+#if defined(Q_OS_LINUX) || defined(Q_OS_WIN)
+// Frame rates worth offering out of a camera that reports a continuous range
+// rather than a list of what it supports. Such a camera will take anything in
+// between, so this is only a question of what to put in a dropdown.
+static const int kStandardFrameRates[] = { 60, 50, 30, 25, 24, 20, 15, 10, 5 };
+
+// Sizes offered for the same reason, out of a camera that reports a range of
+// sizes rather than a list. Ones outside the range, or that the reported step
+// does not land on, are dropped by the caller.
+static const struct
+{
+ int width;
+ int height;
+} kStandardSizes[] =
+{
+ { 1920, 1080 }, { 1280, 720 }, { 1024, 768 }, { 800, 600 },
+ { 848, 480 }, { 640, 480 }, { 640, 360 }, { 424, 240 },
+ { 352, 288 }, { 320, 240 }, { 176, 144 }, { 160, 120 }
+};
+
+// Largest picture first, and highest frame rate first within each.
+static bool captureFormatLessThan(const VideoCaptureFormat &a, const VideoCaptureFormat &b)
+{
+ const qint64 areaA = (qint64)a.width * a.height;
+ const qint64 areaB = (qint64)b.width * b.height;
+
+ if (areaA != areaB)
+ return areaA > areaB;
+
+ return a.width > b.width;
+}
+#endif
+
#ifdef HAVE_VAAPI
// Picks the GPU pixel format out of what the decoder offers. Returning the
// first entry instead - which is what happens when VAAPI is not among them -
@@ -166,10 +291,211 @@ QStringList VideoDevice::CompressedFormats(const QString &device)
return result;
}
+#ifdef Q_OS_LINUX
+// Every picture size the camera offers in one pixel format.
+//
+// A camera answers this in one of two ways: a list of the sizes it has, or a
+// range it will scale anything within. There is no list to show for the
+// second, so the standard sizes that fall inside the range stand in for one.
+static QList<QPair<int, int> > v4l2FrameSizes(int fd, unsigned int pixelformat)
+{
+ QList<QPair<int, int> > sizes;
+ struct v4l2_frmsizeenum fs;
+
+ memset(&fs, 0, sizeof fs);
+ fs.pixel_format = pixelformat;
+
+ for (fs.index = 0; ::ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &fs) == 0; fs.index++)
+ {
+ if (fs.type == V4L2_FRMSIZE_TYPE_DISCRETE)
+ {
+ sizes << qMakePair((int)fs.discrete.width, (int)fs.discrete.height);
+ continue;
+ }
+
+ for (unsigned int i = 0; i < sizeof(kStandardSizes) / sizeof(kStandardSizes[0]); i++)
+ {
+ const unsigned int w = kStandardSizes[i].width;
+ const unsigned int h = kStandardSizes[i].height;
+
+ if (w < fs.stepwise.min_width || w > fs.stepwise.max_width ||
+ h < fs.stepwise.min_height || h > fs.stepwise.max_height)
+ continue;
+
+ // A continuous camera reports a step of one, so this only ever
+ // excludes anything on a genuinely stepwise one.
+ if (fs.stepwise.step_width > 1 && (w - fs.stepwise.min_width) % fs.stepwise.step_width)
+ continue;
+
+ if (fs.stepwise.step_height > 1 && (h - fs.stepwise.min_height) % fs.stepwise.step_height)
+ continue;
+
+ sizes << qMakePair((int)w, (int)h);
+ }
+
+ // A non-discrete answer is a single entry describing the whole range;
+ // there is no index 1 to ask for.
+ break;
+ }
+
+ return sizes;
+}
+
+// v4l2 reports frame intervals - seconds per frame - and we want the rate, so
+// every one of these is a reciprocal.
+static double v4l2IntervalToFps(const struct v4l2_fract &f)
+{
+ if (f.numerator == 0 || f.denominator == 0)
+ return 0.0;
+
+ return (double)f.denominator / (double)f.numerator;
+}
+
+// The frame rates the camera offers for one pixel format at one size, highest
+// first. Same two shapes of answer as the sizes above, handled the same way.
+static QList<int> v4l2FrameRates(int fd, unsigned int pixelformat, int width, int height)
+{
+ QList<int> rates;
+ struct v4l2_frmivalenum fi;
+
+ memset(&fi, 0, sizeof fi);
+ fi.pixel_format = pixelformat;
+ fi.width = width;
+ fi.height = height;
+
+ for (fi.index = 0; ::ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &fi) == 0; fi.index++)
+ {
+ if (fi.type == V4L2_FRMIVAL_TYPE_DISCRETE)
+ {
+ int fps = qRound(v4l2IntervalToFps(fi.discrete));
+
+ if (fps > 0 && !rates.contains(fps))
+ rates << fps;
+
+ continue;
+ }
+
+ // The longest interval is the lowest rate, hence the crossed over
+ // names here.
+ const double minFps = v4l2IntervalToFps(fi.stepwise.max);
+ const double maxFps = v4l2IntervalToFps(fi.stepwise.min);
+
+ for (unsigned int i = 0; i < sizeof(kStandardFrameRates) / sizeof(kStandardFrameRates[0]); i++)
+ {
+ const int fps = kStandardFrameRates[i];
+
+ if (fps >= minFps && fps <= maxFps && !rates.contains(fps))
+ rates << fps;
+ }
+
+ break;
+ }
+
+ std::sort(rates.begin(), rates.end(), std::greater<int>());
+
+ return rates;
+}
+#endif
+
+QList<VideoCaptureFormat> VideoDevice::CaptureFormats(const QString &device)
+{
+ QList<VideoCaptureFormat> result;
+
+#ifdef Q_OS_LINUX
+ int fd = ::open(device.toLatin1().constData(), O_RDWR | O_NONBLOCK);
+ if (fd < 0)
+ return result;
+
+ // Keyed on the size, because a camera reports its sizes once per pixel
+ // format and the same size usually comes back several times over. What
+ // differs between those is the format and the frame rates available in
+ // it, so both are merged into the one entry for the size.
+ QMap<QPair<int, int>, VideoCaptureFormat> bySize;
+
+ struct v4l2_fmtdesc fmt;
+
+ memset(&fmt, 0, sizeof fmt);
+ fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+
+ for (fmt.index = 0; ::ioctl(fd, VIDIOC_ENUM_FMT, &fmt) == 0; fmt.index++)
+ {
+ // What this pixel format is called in a 0x75 message, empty for an
+ // uncompressed one and for a compressed one we would not forward.
+ QString codec;
+
+ for (unsigned int i = 0; i < sizeof(kCompressedFormatPreference) / sizeof(kCompressedFormatPreference[0]); i++)
+ {
+ if (kCompressedFormatPreference[i].v4l2_pixelformat == fmt.pixelformat)
+ {
+ codec = QLatin1String(kCompressedFormatPreference[i].ffmpeg_codec);
+ break;
+ }
+ }
+
+ QList<QPair<int, int> > sizes = v4l2FrameSizes(fd, fmt.pixelformat);
+
+ for (int s = 0; s < sizes.size(); s++)
+ {
+ VideoCaptureFormat &entry = bySize[sizes.at(s)];
+
+ entry.width = sizes.at(s).first;
+ entry.height = sizes.at(s).second;
+
+ if (!codec.isEmpty() && !entry.compressedFormats.contains(codec))
+ entry.compressedFormats << codec;
+
+ QList<int> rates = v4l2FrameRates(fd, fmt.pixelformat, entry.width, entry.height);
+
+ for (int r = 0; r < rates.size(); r++)
+ if (!entry.frameRates.contains(rates.at(r)))
+ entry.frameRates << rates.at(r);
+ }
+ }
+
+ ::close(fd);
+
+ for (QMap<QPair<int, int>, VideoCaptureFormat>::iterator it = bySize.begin(); it != bySize.end(); ++it)
+ {
+ VideoCaptureFormat entry = it.value();
+
+ // The formats were collected in whatever order the camera listed them
+ // in; report them in preference order, the same one
+ // CompressedFormats() and VideoDevice::Capture() work in.
+ QStringList ordered;
+
+ for (unsigned int i = 0; i < sizeof(kCompressedFormatPreference) / sizeof(kCompressedFormatPreference[0]); i++)
+ {
+ QString codec = QLatin1String(kCompressedFormatPreference[i].ffmpeg_codec);
+
+ if (entry.compressedFormats.contains(codec) && !ordered.contains(codec))
+ ordered << codec;
+ }
+
+ entry.compressedFormats = ordered;
+
+ std::sort(entry.frameRates.begin(), entry.frameRates.end(), std::greater<int>());
+
+ result << entry;
+ }
+
+ std::sort(result.begin(), result.end(), captureFormatLessThan);
+#elif defined(Q_OS_WIN)
+ result = pica_dshow_capture_formats(device);
+#else
+ Q_UNUSED(device);
+#endif
+
+ return result;
+}
+
VideoFrameAssembler::VideoFrameAssembler()
- : m_timestamp(0), m_inProgress(false), m_lastSeq(0),
- m_lastWasFrameEnd(false), m_haveLastSeq(false)
+ : m_timestamp(0), m_lastSeq(0)
{
+ // Through reset() rather than an initialiser list of its own: a fresh
+ // assembler and one that has been reset are the same thing, and stating
+ // that start-of-stream state twice is how the two came to disagree - the
+ // constructor's copy is what the first call after launch actually uses.
+ reset();
}
void VideoFrameAssembler::reset()
@@ -177,9 +503,24 @@ void VideoFrameAssembler::reset()
m_buffer.clear();
m_inProgress = false;
m_haveLastSeq = false;
- // The first fragment received is as likely to be the middle of a frame as
- // its beginning, so assembly only starts after the first frame boundary.
- m_lastWasFrameEnd = false;
+
+ // The next fragment is taken to begin a frame, rather than to be the
+ // middle of one whose start was lost.
+ //
+ // Every caller resets a stream that is about to start, never one being
+ // joined in flight - a call resets before the first packet can arrive, and
+ // the switch to a mediac2c connection deliberately does not reset, since
+ // media keeps arriving over either transport for the whole call. So the
+ // first fragment after this really is the first one the sender sent.
+ //
+ // Assuming the opposite costs the stream's first frame, and that frame is
+ // the one that matters: with a camera's own compressed stream being
+ // forwarded untouched, it carries the parameter sets and the only keyframe
+ // the camera will produce for the next ten seconds, so losing it means ten
+ // seconds of black. If this assumption is ever wrong the cost is one
+ // truncated frame that the decoder rejects, and the next frame boundary
+ // resynchronizes as usual.
+ m_lastWasFrameEnd = true;
}
QByteArray VideoFrameAssembler::addFragment(quint16 seq_num, quint32 timestamp, const QByteArray &data)
@@ -242,9 +583,11 @@ QByteArray VideoFrameAssembler::addFragment(quint16 seq_num, quint32 timestamp,
}
VideoDevice::VideoDevice(QObject *parent)
- : QObject(parent), m_width(640), m_height(480), m_preferCompressed(false),
+ : QObject(parent), m_width(kDefaultCaptureWidth), m_height(kDefaultCaptureHeight),
+ m_frameRate(kDefaultCaptureFrameRate), m_bitrate(kDefaultVideoBitrateKbps * 1000),
+ m_preferCompressed(false),
m_useVaapi(false), m_useVaapiRender(false), m_abort(0),
- m_maxFragmentSize(kMaxFragmentSize)
+ m_maxFragmentSize(kMaxFragmentSize), m_decoderStarted(0)
{
}
@@ -261,12 +604,16 @@ VideoDevice::~VideoDevice()
Close();
}
-void VideoDevice::configureCapture(QString deviceName, int width, int height,
- bool preferCompressed, bool useVaapi)
+void VideoDevice::configureCapture(QString deviceName, int width, int height, int frameRate,
+ bool preferCompressed, QString codec, int bitrate,
+ bool useVaapi)
{
m_deviceName = deviceName;
m_width = width;
m_height = height;
+ m_frameRate = frameRate > 0 ? frameRate : kDefaultCaptureFrameRate;
+ m_codec = codec;
+ m_bitrate = bitrate > 0 ? bitrate : kDefaultVideoBitrateKbps * 1000;
m_preferCompressed = preferCompressed;
m_useVaapi = useVaapi;
}
@@ -284,6 +631,9 @@ void VideoDevice::configurePlayback(QString codec, int width, int height,
// A renderer that failed during an earlier call says nothing about this
// one - it may well be a different window.
m_hwRenderDisabled.storeRelaxed(0);
+ // A fresh decode session, so the startup queue depth applies again until
+ // this one's loop is consuming - see enqueueFrame().
+ m_decoderStarted.storeRelaxed(0);
}
void VideoDevice::disableHardwareRendering()
@@ -295,7 +645,10 @@ void VideoDevice::enqueueFrame(QByteArray encodedFrame)
{
QMutexLocker locker(&m_queueMutex);
- while (m_frameQueue.size() >= kMaxQueueDepth)
+ // Deeper while the decoder is still starting up - see kStartupQueueDepth.
+ const int depth = m_decoderStarted.loadRelaxed() ? kMaxQueueDepth : kStartupQueueDepth;
+
+ while (m_frameQueue.size() >= depth)
m_frameQueue.dequeue();
m_frameQueue.enqueue(encodedFrame);
@@ -414,8 +767,18 @@ void VideoDevice::Capture()
QStringList attempts;
if (m_preferCompressed)
+ {
attempts = CompressedFormats(m_deviceName);
+ // The configured codec first when the camera can produce it itself:
+ // forwarding that one means the peer receives exactly what was asked
+ // for, rather than whichever compressed format the camera happened to
+ // rank highest. The rest keep their preference order behind it, since
+ // forwarding any of them still beats decoding and re-encoding.
+ if (attempts.removeAll(m_codec) > 0)
+ attempts.prepend(m_codec);
+ }
+
attempts << QString();
// Listing a format and being able to produce it at a given size and frame
@@ -452,7 +815,7 @@ void VideoDevice::Capture()
ret = openCamera(ifmt, deviceUtf8,
kConstraints[c].size ? m_width : 0,
kConstraints[c].size ? m_height : 0,
- kConstraints[c].rate ? kFrameRate : 0,
+ kConstraints[c].rate ? m_frameRate : 0,
attempts.at(a), &ifmt_ctx);
if (ret >= 0)
@@ -467,7 +830,7 @@ void VideoDevice::Capture()
if (!kConstraints[c].size)
asked = QStringLiteral("any size or rate");
else if (kConstraints[c].rate)
- asked = QString("%1x%2 @%3").arg(m_width).arg(m_height).arg(kFrameRate);
+ asked = QString("%1x%2 @%3").arg(m_width).arg(m_height).arg(m_frameRate);
else
asked = QString("%1x%2 at any rate").arg(m_width).arg(m_height);
@@ -604,7 +967,7 @@ void VideoDevice::runPassthroughLoop(AVFormatContext *ifmt_ctx, AVStream *in_st,
// one that was asked for - and when the open had to fall back to letting the
// camera choose, was not asked for at all. Encoding at a rate the frames do
// not arrive at gives a stream whose timestamps disagree with reality.
-static AVRational cameraFrameRate(AVStream *in_st)
+static AVRational cameraFrameRate(AVStream *in_st, int fallbackRate)
{
AVRational fr = in_st->avg_frame_rate;
@@ -612,14 +975,19 @@ static AVRational cameraFrameRate(AVStream *in_st)
fr = in_st->r_frame_rate;
if (fr.num <= 0 || fr.den <= 0)
- fr = AVRational{ kFrameRate, 1 };
+ fr = AVRational{ fallbackRate, 1 };
return fr;
}
void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
{
- const AVRational frameRate = cameraFrameRate(in_st);
+ const AVRational frameRate = cameraFrameRate(in_st, m_frameRate);
+
+ // Which encoder produces the configured codec. Looked up once: the GPU
+ // attempt and the software fallback below are two encoders for the same
+ // codec, so the peer is told the same thing either way.
+ const int encIdx = videoEncoderIndex(m_codec);
// Two seconds between keyframes. A receiver that joins late or loses a
// packet waits this long for a picture, so it is a latency figure as much
@@ -657,7 +1025,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
#ifdef HAVE_VAAPI
if (m_useVaapi && VaapiContext::isAvailable())
{
- enc = avcodec_find_encoder_by_name("h264_vaapi");
+ enc = avcodec_find_encoder_by_name(kVideoEncoders[encIdx].vaapiEncoder);
AVBufferRef *hw_device_ref = enc ? VaapiContext::deviceRef() : nullptr;
if (hw_device_ref)
@@ -684,18 +1052,18 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
enc_ctx->pix_fmt = AV_PIX_FMT_VAAPI;
enc_ctx->time_base = av_inv_q(frameRate);
enc_ctx->framerate = frameRate;
- enc_ctx->bit_rate = kBitrate;
+ enc_ctx->bit_rate = m_bitrate;
enc_ctx->gop_size = gopSize;
enc_ctx->max_b_frames = 0;
enc_ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
- // No preset/tune here: those are x264's own options and
- // mean nothing to a VAAPI encoder.
+ // No preset/tune here: those belong to the software
+ // encoders and mean nothing to a VAAPI one.
if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) >= 0)
hardware = true;
else
- qWarning() << QString("VAAPI H.264 encoder would not open (%1), encoding in software")
- .arg(ff_errstr(ret));
+ qWarning() << QString("The VAAPI %1 encoder would not open (%2), encoding in software")
+ .arg(QLatin1String(kVideoEncoders[encIdx].codec), ff_errstr(ret));
}
else
{
@@ -711,7 +1079,8 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
}
else
{
- qWarning() << "FFmpeg has no h264_vaapi encoder, encoding in software";
+ qWarning() << QString("FFmpeg has no %1 encoder, encoding in software")
+ .arg(QLatin1String(kVideoEncoders[encIdx].vaapiEncoder));
}
if (!hardware)
@@ -723,27 +1092,35 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
}
#endif
+ // Named rather than taken from enc->name because the VAAPI branch above
+ // may have left a different encoder in enc; filled in below for the
+ // software path, which is the only one the private options apply to.
+ QString swEncoderName;
+
if (!hardware)
{
- enc = avcodec_find_encoder_by_name("libx264");
+ enc = avcodec_find_encoder_by_name(kVideoEncoders[encIdx].swEncoder);
if (!enc)
- enc = avcodec_find_encoder(AV_CODEC_ID_H264);
+ enc = avcodec_find_encoder(kVideoEncoders[encIdx].id);
enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr;
if (!enc || !enc_ctx)
{
- QString msg = "H.264 encoder not available";
+ QString msg = QString("No %1 encoder available").arg(QLatin1String(kVideoEncoders[encIdx].codec));
qWarning() << msg;
emit errorOccurred(msg);
+ if (enc_ctx) avcodec_free_context(&enc_ctx);
avcodec_free_context(&dec_ctx);
return;
}
+ swEncoderName = QLatin1String(enc->name);
+
enc_ctx->width = m_width;
enc_ctx->height = m_height;
enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
enc_ctx->time_base = av_inv_q(frameRate);
enc_ctx->framerate = frameRate;
- enc_ctx->bit_rate = kBitrate;
+ enc_ctx->bit_rate = m_bitrate;
enc_ctx->gop_size = gopSize;
// B-frames reorder output, which would cost latency in a live call.
enc_ctx->max_b_frames = 0;
@@ -751,19 +1128,11 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
// Note the absence of AV_CODEC_FLAG_GLOBAL_HEADER: we want the parameter
// sets repeated in-band with every keyframe, since the receiving side has
// no out-of-band way to learn them and may start decoding mid-stream.
- av_opt_set(enc_ctx->priv_data, "preset", "veryfast", 0);
- av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0);
-
- // Keeps every slice within one datagram of a mediac2c connection, so
- // that a lost fragment costs a slice of the picture rather than the
- // whole frame. Harmless when the media goes over TCP instead - it
- // only splits frames into more NAL units than strictly necessary.
- av_opt_set(enc_ctx->priv_data, "x264opts",
- QString("slice-max-size=%1").arg(kSliceMaxSize).toUtf8().constData(), 0);
+ applyLiveEncoderOptions(enc_ctx, swEncoderName, kSliceMaxSize);
if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) < 0)
{
- QString msg = QString("Could not open H.264 encoder: %1").arg(ff_errstr(ret));
+ QString msg = QString("Could not open the %1 encoder: %2").arg(swEncoderName, ff_errstr(ret));
qWarning() << msg;
emit errorOccurred(msg);
avcodec_free_context(&enc_ctx);
@@ -772,8 +1141,9 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
}
}
- emit accelerationInUse(hardware ? QStringLiteral("GPU encoding (VAAPI)")
- : QStringLiteral("CPU encoding (libx264)"));
+ emit accelerationInUse(hardware
+ ? QString("GPU encoding (VAAPI %1)").arg(QLatin1String(kVideoEncoders[encIdx].codec))
+ : QString("CPU encoding (%1)").arg(swEncoderName));
// Everything the camera sends is scaled to the encoder's size and re-encoded,
// so this is what the peer will receive regardless of what the camera
@@ -910,7 +1280,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
{
if (!loggedEncodeError)
{
- qWarning() << "H.264 encode failed";
+ qWarning() << QString("%1 encode failed").arg(QLatin1String(avcodec_get_name(enc_ctx->codec_id)));
loggedEncodeError = true;
}
continue;
@@ -1011,6 +1381,12 @@ void VideoDevice::Play()
continue;
frameData = m_frameQueue.dequeue();
+
+ // The codec is open and the loop is consuming, so the backlog held
+ // for startup is no longer wanted: from here on the newest frame
+ // wins. Set under the queue mutex so enqueueFrame() cannot read it
+ // while this frame is being taken.
+ m_decoderStarted.storeRelaxed(1);
}
AVPacket *in_pkt = av_packet_alloc();
@@ -1027,7 +1403,7 @@ void VideoDevice::Play()
{
if (!loggedDecodeError)
{
- qWarning() << QString("H.264 decode failed: %1").arg(ff_errstr(ret));
+ qWarning() << QString("%1 decode failed: %2").arg(m_codec, ff_errstr(ret));
loggedDecodeError = true;
}
continue;
@@ -1211,7 +1587,6 @@ QList<MediaDeviceInfo> VideoDevice::Enumerate(enum MediaDeviceStreamDirection di
if (!(cap.device_caps & V4L2_CAP_VIDEO_CAPTURE))
continue;
d.index = index++;
- d.compressedFormats = CompressedFormats(d.device);
result << d;
}
diff --git a/pica-client/videodevice.h b/pica-client/videodevice.h
index 58eaef9..fe218ec 100644
--- a/pica-client/videodevice.h
+++ b/pica-client/videodevice.h
@@ -63,6 +63,29 @@ private:
bool m_haveLastSeq;
};
+// One picture size a camera can deliver, and what it can deliver it in.
+// Reported by VideoDevice::CaptureFormats() and shown by the settings dialog's
+// resolution and frame rate lists.
+struct VideoCaptureFormat
+{
+ int width;
+ int height;
+
+ // Frame rates the camera offers at this size, in whole frames per second,
+ // highest first. Empty when the camera would not say - which is not the
+ // same as "none": the capture code can still open it and let the camera
+ // pick, see VideoDevice::Capture().
+ QList<int> frameRates;
+
+ // Compressed formats available at this size, as FFmpeg codec names, most
+ // preferred first. The same names and the same order as
+ // CompressedFormats() reports for the whole camera, but per size - a
+ // camera routinely offers MJPEG at its larger sizes only, which is exactly
+ // what the "prefer compressed formats" setting turns on and what makes the
+ // choice of size worth showing alongside.
+ QStringList compressedFormats;
+};
+
// Video counterpart of AudioDevice: a single instance is dedicated to one
// direction - either capture+encode (Capture(), reading from a local camera)
// or decode (Play(), fed from the network via enqueueFrame()). Each instance
@@ -83,18 +106,31 @@ public:
VideoDevice(QObject *parent = nullptr);
~VideoDevice();
+ // width, height and frameRate are what the camera is asked for; a camera
+ // that will not honour them is opened with them loosened rather than not
+ // at all, see Capture().
+ //
// preferCompressed asks for the camera's own compressed stream to be
// forwarded untouched when it offers one, saving the cost of decoding and
// re-encoding every frame. The codec that ends up being used is only
// known once the camera has been opened, and is reported by
// captureStarted() - it is not necessarily the one requested here.
//
+ // codec is the FFmpeg name of the codec to encode to - "h264", "hevc" or
+ // "vp9" - and bitrate the rate to encode at, in bits per second. Both
+ // apply to frames this class encodes itself; a camera stream being
+ // forwarded untouched is already encoded and neither has anything to act
+ // on. codec is preferred over the camera's other compressed formats when
+ // looking for one to forward, so choosing a codec the camera can produce
+ // itself gets that one rather than merely a compatible one.
+ //
// useVaapi asks for the frames to be encoded on the GPU. It is a request,
// not a guarantee: without VAAPI support built in, without usable
// hardware, or if the hardware encoder will not open, capture falls back
// to encoding in software.
- Q_INVOKABLE void configureCapture(QString deviceName, int width, int height,
- bool preferCompressed, bool useVaapi);
+ Q_INVOKABLE void configureCapture(QString deviceName, int width, int height, int frameRate,
+ bool preferCompressed, QString codec, int bitrate,
+ bool useVaapi);
// useVaapi decodes on the GPU. useVaapiRender additionally keeps the
// decoded frames there and emits them through hwFrameReady() instead of
@@ -185,11 +221,20 @@ public:
// are decoded and re-encoded as usual.
static QStringList CompressedFormats(const QString &device);
+ // Every picture size the given camera can deliver, largest first, with the
+ // frame rates and compressed formats available at each. Empty on a
+ // platform where the camera cannot be asked - the caller then falls back
+ // to offering a set of common sizes, which the camera is free to refuse
+ // the same way it may refuse any other request.
+ static QList<VideoCaptureFormat> CaptureFormats(const QString &device);
+
private:
QString m_deviceName;
QString m_codec;
int m_width;
int m_height;
+ int m_frameRate;
+ int m_bitrate;
bool m_preferCompressed;
bool m_useVaapi;
bool m_useVaapiRender;
@@ -206,8 +251,27 @@ private:
QMutex m_queueMutex;
QWaitCondition m_queueCond;
QQueue<QByteArray> m_frameQueue;
+
+ // Steady state: the newest frame wins, because for live video showing the
+ // current picture matters more than showing every picture.
static const int kMaxQueueDepth = 2;
+ // Until the decode loop has taken its first frame it is not behind - it is
+ // still opening the codec, which happens only after the peer's 0x75 has
+ // been processed, by which time its packets are already arriving. Applying
+ // the rule above during that window drops the stream's first keyframe and
+ // then the frames that follow it, which costs every picture until the next
+ // keyframe: with a camera's own stream forwarded untouched that can be ten
+ // seconds away. So a bounded backlog is held instead - a couple of seconds
+ // at a typical call frame rate - and the queue reverts to the depth above
+ // as soon as the decoder is actually consuming.
+ static const int kStartupQueueDepth = 30;
+
+ // Whether the decode loop has taken a frame yet; see enqueueFrame(). Read
+ // and written from both the decoding thread and whichever thread delivers
+ // packets, hence atomic.
+ QAtomicInt m_decoderStarted;
+
// Sends one encoded frame out as one or more protocol sized fragments.
void emitFragments(const unsigned char *data, int size);
diff --git a/pica-client/videodshow.cpp b/pica-client/videodshow.cpp
index 150e8a6..9b389e9 100644
--- a/pica-client/videodshow.cpp
+++ b/pica-client/videodshow.cpp
@@ -15,6 +15,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "mediadevice.h"
+#include "videodevice.h"
#include <QtGlobal>
@@ -22,9 +23,13 @@
#include <QDebug>
#include <QList>
+#include <QMap>
+#include <QPair>
#include <QString>
#include <QStringList>
#include <QVector>
+#include <algorithm>
+#include <functional>
// INITGUID before the DirectShow headers so CLSID_SystemDeviceEnum and the
// rest get defined here rather than needing strmiids at link time. The GUIDs
@@ -301,6 +306,232 @@ QStringList pica_dshow_compressed_formats(const QString &device)
return result;
}
+// Frame rates worth offering out of a pin that reports a range of frame
+// intervals rather than one fixed rate. Same list, and the same reasoning, as
+// kStandardFrameRates in videodevice.cpp.
+static const int kDshowStandardFrameRates[] = { 60, 50, 30, 25, 24, 20, 15, 10, 5 };
+
+// DirectShow measures a frame interval in 100 nanosecond units, and we want
+// the rate, so this is a reciprocal like the v4l2 one.
+static double intervalToFps(LONGLONG interval)
+{
+ if (interval <= 0)
+ return 0.0;
+
+ return 10000000.0 / (double)interval;
+}
+
+// Largest picture first - the same order VideoDevice::CaptureFormats() reports
+// on Linux, so the settings dialog's list reads the same on both.
+static bool dshowFormatLessThan(const VideoCaptureFormat &a, const VideoCaptureFormat &b)
+{
+ const qint64 areaA = (qint64)a.width * a.height;
+ const qint64 areaB = (qint64)b.width * b.height;
+
+ if (areaA != areaB)
+ return areaA > areaB;
+
+ return a.width > b.width;
+}
+
+// Every picture size the camera can deliver, with the frame rates and
+// compressed formats available at each.
+//
+// The counterpart of the v4l2 VIDIOC_ENUM_FRAMESIZES/VIDIOC_ENUM_FRAMEINTERVALS
+// walk in VideoDevice::CaptureFormats(), and it comes off the same
+// IAMStreamConfig enumeration pica_dshow_compressed_formats() uses: each media
+// type carries a size and a FourCC, and the VIDEO_STREAM_CONFIG_CAPS alongside
+// it carries the range of frame intervals that size can be delivered at.
+//
+// The same caveat applies as there: this instantiates the capture filter, so
+// an empty list can mean "cannot be opened at all" as much as "offers
+// nothing". Either way the caller falls back to a set of common sizes.
+QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device)
+{
+ QList<VideoCaptureFormat> result;
+
+ if (device.isEmpty())
+ return result;
+
+ bool didInit = false;
+ HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
+ if (SUCCEEDED(hr))
+ didInit = true;
+ else if (hr != RPC_E_CHANGED_MODE)
+ return result;
+
+ IMoniker *moniker = findCamera(device);
+ if (!moniker)
+ {
+ if (didInit)
+ CoUninitialize();
+ return result;
+ }
+
+ IBaseFilter *filter = nullptr;
+ hr = moniker->BindToObject(nullptr, nullptr, IID_IBaseFilter, (void **)&filter);
+ moniker->Release();
+
+ if (FAILED(hr) || !filter)
+ {
+ qWarning() << "DirectShow: could not instantiate" << device
+ << "to ask what sizes it offers";
+ if (didInit)
+ CoUninitialize();
+ return result;
+ }
+
+ // Keyed on the size, because a camera lists one media type per size and
+ // format combination and the same size comes back once per format.
+ QMap<QPair<int, int>, VideoCaptureFormat> bySize;
+
+ IEnumPins *enumPins = nullptr;
+ if (SUCCEEDED(filter->EnumPins(&enumPins)) && enumPins)
+ {
+ IPin *pin = nullptr;
+
+ while (enumPins->Next(1, &pin, nullptr) == S_OK)
+ {
+ PIN_DIRECTION dir;
+ IAMStreamConfig *config = nullptr;
+
+ if (SUCCEEDED(pin->QueryDirection(&dir)) && dir == PINDIR_OUTPUT &&
+ SUCCEEDED(pin->QueryInterface(IID_IAMStreamConfig, (void **)&config)) && config)
+ {
+ int count = 0, size = 0;
+
+ if (SUCCEEDED(config->GetNumberOfCapabilities(&count, &size)) &&
+ size >= (int)sizeof(VIDEO_STREAM_CONFIG_CAPS))
+ {
+ QVector<quint8> caps(size);
+
+ for (int i = 0; i < count; i++)
+ {
+ AM_MEDIA_TYPE *mt = nullptr;
+ if (FAILED(config->GetStreamCaps(i, &mt, caps.data())) || !mt)
+ continue;
+
+ const BITMAPINFOHEADER *bih = nullptr;
+ // The nominal interval of this particular media type,
+ // as opposed to the range in the caps below.
+ LONGLONG avgTimePerFrame = 0;
+
+ if (mt->formattype == FORMAT_VideoInfo &&
+ mt->cbFormat >= sizeof(VIDEOINFOHEADER))
+ {
+ const VIDEOINFOHEADER *vih = (const VIDEOINFOHEADER *)mt->pbFormat;
+ bih = &vih->bmiHeader;
+ avgTimePerFrame = vih->AvgTimePerFrame;
+ }
+ else if (mt->formattype == FORMAT_VideoInfo2 &&
+ mt->cbFormat >= sizeof(VIDEOINFOHEADER2))
+ {
+ const VIDEOINFOHEADER2 *vih = (const VIDEOINFOHEADER2 *)mt->pbFormat;
+ bih = &vih->bmiHeader;
+ avgTimePerFrame = vih->AvgTimePerFrame;
+ }
+
+ if (!bih || bih->biWidth <= 0 || bih->biHeight == 0)
+ {
+ freeMediaType(mt);
+ continue;
+ }
+
+ // A bottom-up DIB states its height negative; the
+ // picture is the same size either way.
+ const int width = (int)bih->biWidth;
+ const int height = bih->biHeight < 0 ? (int)-bih->biHeight : (int)bih->biHeight;
+
+ VideoCaptureFormat &entry = bySize[qMakePair(width, height)];
+ entry.width = width;
+ entry.height = height;
+
+ const QString fourcc = fourccToString(bih->biCompression);
+
+ for (unsigned int f = 0; f < sizeof(kDshowCompressedFormats) / sizeof(kDshowCompressedFormats[0]); f++)
+ {
+ if (fourcc != QLatin1String(kDshowCompressedFormats[f].fourcc))
+ continue;
+
+ QString codec = QLatin1String(kDshowCompressedFormats[f].ffmpeg_codec);
+
+ if (!entry.compressedFormats.contains(codec))
+ entry.compressedFormats << codec;
+
+ break;
+ }
+
+ const VIDEO_STREAM_CONFIG_CAPS *scc = (const VIDEO_STREAM_CONFIG_CAPS *)caps.constData();
+
+ // The shortest interval is the highest rate, hence the
+ // crossed over names.
+ const double maxFps = intervalToFps(scc->MinFrameInterval);
+ const double minFps = intervalToFps(scc->MaxFrameInterval);
+
+ int nominal = qRound(intervalToFps(avgTimePerFrame));
+
+ if (nominal > 0 && !entry.frameRates.contains(nominal))
+ entry.frameRates << nominal;
+
+ // A pin that accepts a range of intervals will take
+ // anything within it, so there is no list to show -
+ // the standard rates inside the range stand in for one,
+ // exactly as on the v4l2 side.
+ for (unsigned int r = 0; r < sizeof(kDshowStandardFrameRates) / sizeof(kDshowStandardFrameRates[0]); r++)
+ {
+ const int fps = kDshowStandardFrameRates[r];
+
+ if (fps >= minFps && fps <= maxFps && !entry.frameRates.contains(fps))
+ entry.frameRates << fps;
+ }
+
+ freeMediaType(mt);
+ }
+ }
+ }
+
+ if (config)
+ config->Release();
+ pin->Release();
+ }
+
+ enumPins->Release();
+ }
+
+ filter->Release();
+
+ for (QMap<QPair<int, int>, VideoCaptureFormat>::iterator it = bySize.begin(); it != bySize.end(); ++it)
+ {
+ VideoCaptureFormat entry = it.value();
+
+ // Report the formats in our preference order rather than in the order
+ // the camera happened to list them, the same way
+ // pica_dshow_compressed_formats() does.
+ QStringList ordered;
+
+ for (unsigned int f = 0; f < sizeof(kDshowCompressedFormats) / sizeof(kDshowCompressedFormats[0]); f++)
+ {
+ QString codec = QLatin1String(kDshowCompressedFormats[f].ffmpeg_codec);
+
+ if (entry.compressedFormats.contains(codec) && !ordered.contains(codec))
+ ordered << codec;
+ }
+
+ entry.compressedFormats = ordered;
+
+ std::sort(entry.frameRates.begin(), entry.frameRates.end(), std::greater<int>());
+
+ result << entry;
+ }
+
+ std::sort(result.begin(), result.end(), dshowFormatLessThan);
+
+ if (didInit)
+ CoUninitialize();
+
+ return result;
+}
+
// Lists the cameras DirectShow knows about.
//
// Capture itself goes through FFmpeg's "dshow" demuxer rather than through
@@ -413,11 +644,10 @@ QList<MediaDeviceInfo> pica_enumerate_dshow_video()
d.index = index++;
- // Left empty deliberately. compressedFormats drives the "prefer
- // compressed formats" setting, and working out what a DirectShow
- // camera can deliver means walking its output pin's media types
- // through IAMStreamConfig - a good deal more work than the v4l2 call
- // that fills this in on Linux, and not needed to get a picture.
+ // Nothing here about what the camera can deliver: that means walking
+ // its output pin's media types, which pica_dshow_capture_formats()
+ // does once a camera has actually been selected rather than for every
+ // camera on the machine.
result << d;
}