summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--pica-client/videodevice.cpp233
-rw-r--r--pica-client/videodshow.cpp261
2 files changed, 438 insertions, 56 deletions
diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp
index 8ca5cdc..5b7d5fb 100644
--- a/pica-client/videodevice.cpp
+++ b/pica-client/videodevice.cpp
@@ -58,10 +58,6 @@ extern "C" {
// the feature plan; a fixed, widely supported set of parameters for now.
static const int kFrameRate = 15;
static const int64_t kBitrate = 600000;
-// Keyframe every ~2 seconds. There is no retransmission or keyframe request
-// machinery yet, so this doubles as how quickly a receiver that missed data
-// can resynchronize.
-static const int kGopSize = kFrameRate * 2;
// Largest amount of encoded data one 0x77 message can carry over a c2c or
// directc2c connection: the protocol's per-message payload cap, minus the
@@ -82,6 +78,14 @@ static QString ff_errstr(int err)
return QString::fromLocal8Bit(buf);
}
+#ifdef Q_OS_WIN
+// Both defined in videodshow.cpp. FFmpeg's dshow demuxer can capture but can
+// neither enumerate the cameras nor say what formats one offers, so both
+// answers come from DirectShow itself.
+QList<MediaDeviceInfo> pica_enumerate_dshow_video();
+QStringList pica_dshow_compressed_formats(const QString &device);
+#endif
+
#ifdef Q_OS_LINUX
// Compressed formats worth forwarding untouched, best first: HEVC gives the
// most picture per byte, H.264 is the widely supported middle ground, and
@@ -153,6 +157,8 @@ QStringList VideoDevice::CompressedFormats(const QString &device)
if (offered.contains(kCompressedFormatPreference[i].v4l2_pixelformat) && !result.contains(codec))
result << codec;
}
+#elif defined(Q_OS_WIN)
+ result = pica_dshow_compressed_formats(device);
#else
Q_UNUSED(device);
#endif
@@ -323,6 +329,61 @@ static QByteArray cameraOpenUrl(const QString &device)
#endif
}
+// One attempt at opening the camera.
+//
+// passthroughCodec empty asks for whatever the camera gives; a width of zero
+// drops the size and frame rate constraints as well. *ifmt_ctx is left null on
+// failure - avformat_open_input() frees and clears it - so the caller can just
+// try again.
+static int openCamera(const AVInputFormat *ifmt, const QByteArray &url,
+ int width, int height, int framerate,
+ const QString &passthroughCodec, AVFormatContext **ifmt_ctx)
+{
+ AVDictionary *opts = nullptr;
+
+ if (width > 0 && height > 0)
+ av_dict_set(&opts, "video_size", QString("%1x%2").arg(width).arg(height).toUtf8().constData(), 0);
+
+ if (framerate > 0)
+ av_dict_set(&opts, "framerate", QString::number(framerate).toUtf8().constData(), 0);
+
+ if (!passthroughCodec.isEmpty())
+ {
+#ifdef Q_OS_WIN
+ // dshow has no input_format option. It picks the compressed stream
+ // from AVFormatContext::video_codec_id instead, which is what
+ // "-vcodec mjpeg" ahead of "-i" sets on the command line - and that
+ // has to be set on a context allocated here, since
+ // avformat_open_input() would otherwise make one only after the point
+ // where the demuxer reads it.
+ const AVCodec *dec = avcodec_find_decoder_by_name(passthroughCodec.toUtf8().constData());
+
+ if (!dec)
+ {
+ av_dict_free(&opts);
+ return AVERROR(EINVAL);
+ }
+
+ *ifmt_ctx = avformat_alloc_context();
+
+ if (!*ifmt_ctx)
+ {
+ av_dict_free(&opts);
+ return AVERROR(ENOMEM);
+ }
+
+ (*ifmt_ctx)->video_codec_id = dec->id;
+#else
+ av_dict_set(&opts, "input_format", passthroughCodec.toUtf8().constData(), 0);
+#endif
+ }
+
+ int ret = avformat_open_input(ifmt_ctx, url.constData(), ifmt, &opts);
+ av_dict_free(&opts);
+
+ return ret;
+}
+
void VideoDevice::Capture()
{
m_abort.storeRelaxed(0);
@@ -347,54 +408,75 @@ void VideoDevice::Capture()
// Ask the camera for a compressed stream when told to and it offers one:
// its packets can then go straight onto the network, with no decoding and
// re-encoding in between. The formats are queried rather than guessed, so
- // this asks for one the camera has already said it can deliver.
- QString passthroughCodec;
+ // these are ones the camera has already said it can deliver, best first.
+ // The empty string on the end is the fallback of taking whatever comes and
+ // re-encoding it.
+ QStringList attempts;
if (m_preferCompressed)
+ attempts = CompressedFormats(m_deviceName);
+
+ attempts << QString();
+
+ // Listing a format and being able to produce it at a given size and frame
+ // rate are three separate claims, and cameras routinely honour the first
+ // without the others - an integrated webcam offering MJPEG but not at
+ // 640x480 15fps is what prompted this. So each candidate is tried with the
+ // constraints loosened a step at a time, giving up the frame rate before
+ // the size and both before giving up the format, since the format is worth
+ // the most: it decides whether anything has to be re-encoded at all.
+ //
+ // Whatever comes back is what gets announced to the peer - see
+ // captureStarted() and AudioVideoCallController::video_capture_started() -
+ // so relaxing these changes what is sent, not what is claimed.
+ static const struct
{
- QStringList formats = CompressedFormats(m_deviceName);
-
- if (!formats.isEmpty())
- passthroughCodec = formats.first();
- }
-
- AVDictionary *opts = nullptr;
- av_dict_set(&opts, "video_size", QString("%1x%2").arg(m_width).arg(m_height).toUtf8().constData(), 0);
- av_dict_set(&opts, "framerate", QString::number(kFrameRate).toUtf8().constData(), 0);
- if (!passthroughCodec.isEmpty())
- av_dict_set(&opts, "input_format", passthroughCodec.toUtf8().constData(), 0);
+ bool size;
+ bool rate;
+ } kConstraints[] =
+ {
+ { true, true },
+ { true, false },
+ { false, false },
+ };
+ QString passthroughCodec;
AVFormatContext *ifmt_ctx = nullptr;
- int ret = avformat_open_input(&ifmt_ctx, deviceUtf8.constData(), ifmt, &opts);
- av_dict_free(&opts);
+ int ret = AVERROR(EINVAL);
+ bool opened = false;
- if (ret < 0 && !passthroughCodec.isEmpty())
+ for (int a = 0; a < attempts.size() && !opened; a++)
{
- // The camera listed the format but will not actually hand it over at
- // this size or frame rate. Fall back to whatever it does give.
- qWarning() << QString("Camera '%1' would not deliver %2 (%3), falling back to re-encoding")
- .arg(m_deviceName, passthroughCodec, ff_errstr(ret));
- passthroughCodec.clear();
-
- opts = nullptr;
- av_dict_set(&opts, "video_size", QString("%1x%2").arg(m_width).arg(m_height).toUtf8().constData(), 0);
- av_dict_set(&opts, "framerate", QString::number(kFrameRate).toUtf8().constData(), 0);
-
- ret = avformat_open_input(&ifmt_ctx, deviceUtf8.constData(), ifmt, &opts);
- av_dict_free(&opts);
- }
+ for (unsigned int c = 0; c < sizeof(kConstraints) / sizeof(kConstraints[0]); c++)
+ {
+ ret = openCamera(ifmt, deviceUtf8,
+ kConstraints[c].size ? m_width : 0,
+ kConstraints[c].size ? m_height : 0,
+ kConstraints[c].rate ? kFrameRate : 0,
+ attempts.at(a), &ifmt_ctx);
- if (ret < 0)
- {
- // Last resort: let the camera pick. DirectShow in particular will
- // refuse a video_size or framerate it cannot produce exactly, rather
- // than settling for the nearest it can, so a camera with an unusual
- // set of modes would otherwise give no picture at all. Whatever comes
- // back is scaled to the negotiated size by the transcode loop anyway.
- qWarning() << QString("Camera '%1' would not open at %2x%3 @%4 (%5), letting it choose")
- .arg(m_deviceName).arg(m_width).arg(m_height).arg(kFrameRate).arg(ff_errstr(ret));
-
- ret = avformat_open_input(&ifmt_ctx, deviceUtf8.constData(), ifmt, nullptr);
+ if (ret >= 0)
+ {
+ passthroughCodec = attempts.at(a);
+ opened = true;
+ break;
+ }
+
+ QString asked;
+
+ if (!kConstraints[c].size)
+ asked = QStringLiteral("any size or rate");
+ else if (kConstraints[c].rate)
+ asked = QString("%1x%2 @%3").arg(m_width).arg(m_height).arg(kFrameRate);
+ else
+ asked = QString("%1x%2 at any rate").arg(m_width).arg(m_height);
+
+ qWarning() << QString("Camera '%1': no %2 at %3 (%4)")
+ .arg(m_deviceName,
+ attempts.at(a).isEmpty() ? QStringLiteral("stream") : attempts.at(a),
+ asked,
+ ff_errstr(ret));
+ }
}
if (ret < 0)
@@ -431,6 +513,26 @@ void VideoDevice::Capture()
AVStream *in_st = ifmt_ctx->streams[0];
+ // Check what actually came back rather than trusting the request. A
+ // camera can list a format and then open in a different one, and asking
+ // for it is spelled differently per demuxer - so the one thing worth
+ // relying on is the stream in front of us. Handing raw frames to the
+ // passthrough loop would put unencoded video on the wire labelled as
+ // something it is not.
+ if (!passthroughCodec.isEmpty())
+ {
+ const AVCodec *wanted = avcodec_find_decoder_by_name(passthroughCodec.toUtf8().constData());
+
+ if (!wanted || in_st->codecpar->codec_id != wanted->id)
+ {
+ qWarning() << QString("Camera '%1' opened as %2 rather than the %3 that was asked for, re-encoding")
+ .arg(m_deviceName,
+ QLatin1String(avcodec_get_name(in_st->codecpar->codec_id)),
+ passthroughCodec);
+ passthroughCodec.clear();
+ }
+ }
+
if (!passthroughCodec.isEmpty())
runPassthroughLoop(ifmt_ctx, in_st, passthroughCodec);
else
@@ -498,8 +600,33 @@ void VideoDevice::runPassthroughLoop(AVFormatContext *ifmt_ctx, AVStream *in_st,
av_packet_free(&pkt);
}
+// The frame rate the camera actually settled on, which is not necessarily the
+// one that was asked for - and when the open had to fall back to letting the
+// camera choose, was not asked for at all. Encoding at a rate the frames do
+// not arrive at gives a stream whose timestamps disagree with reality.
+static AVRational cameraFrameRate(AVStream *in_st)
+{
+ AVRational fr = in_st->avg_frame_rate;
+
+ if (fr.num <= 0 || fr.den <= 0)
+ fr = in_st->r_frame_rate;
+
+ if (fr.num <= 0 || fr.den <= 0)
+ fr = AVRational{ kFrameRate, 1 };
+
+ return fr;
+}
+
void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
{
+ const AVRational frameRate = cameraFrameRate(in_st);
+
+ // Two seconds between keyframes. A receiver that joins late or loses a
+ // packet waits this long for a picture, so it is a latency figure as much
+ // as a bandwidth one - hence deriving it from the real rate rather than
+ // from a fixed frame count.
+ const int gopSize = qMax(1, (int)av_q2d(frameRate) * 2);
+
int ret;
// The camera hands us either raw frames or a compressed stream that is
@@ -555,10 +682,10 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
enc_ctx->width = m_width;
enc_ctx->height = m_height;
enc_ctx->pix_fmt = AV_PIX_FMT_VAAPI;
- enc_ctx->time_base = AVRational{1, kFrameRate};
- enc_ctx->framerate = AVRational{kFrameRate, 1};
+ enc_ctx->time_base = av_inv_q(frameRate);
+ enc_ctx->framerate = frameRate;
enc_ctx->bit_rate = kBitrate;
- enc_ctx->gop_size = kGopSize;
+ enc_ctx->gop_size = gopSize;
enc_ctx->max_b_frames = 0;
enc_ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
@@ -614,10 +741,10 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
enc_ctx->width = m_width;
enc_ctx->height = m_height;
enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
- enc_ctx->time_base = AVRational{1, kFrameRate};
- enc_ctx->framerate = AVRational{kFrameRate, 1};
+ enc_ctx->time_base = av_inv_q(frameRate);
+ enc_ctx->framerate = frameRate;
enc_ctx->bit_rate = kBitrate;
- enc_ctx->gop_size = kGopSize;
+ enc_ctx->gop_size = gopSize;
// B-frames reorder output, which would cost latency in a live call.
enc_ctx->max_b_frames = 0;
@@ -1034,12 +1161,6 @@ void VideoDevice::Play()
avcodec_free_context(&dec_ctx);
}
-#ifdef Q_OS_WIN
-// Defined in videodshow.cpp - FFmpeg's dshow demuxer can capture but cannot
-// enumerate, so the list comes from DirectShow itself.
-QList<MediaDeviceInfo> pica_enumerate_dshow_video();
-#endif
-
QList<MediaDeviceInfo> VideoDevice::Enumerate(enum MediaDeviceStreamDirection dir)
{
QList<MediaDeviceInfo> result;
diff --git a/pica-client/videodshow.cpp b/pica-client/videodshow.cpp
index 394798a..150e8a6 100644
--- a/pica-client/videodshow.cpp
+++ b/pica-client/videodshow.cpp
@@ -23,6 +23,8 @@
#include <QDebug>
#include <QList>
#include <QString>
+#include <QStringList>
+#include <QVector>
// INITGUID before the DirectShow headers so CLSID_SystemDeviceEnum and the
// rest get defined here rather than needing strmiids at link time. The GUIDs
@@ -34,12 +36,271 @@
#include <windows.h>
#include <objbase.h>
#include <dshow.h>
+// VIDEOINFOHEADER2 lives here rather than in dshow.h, which only brings in the
+// FORMAT_VideoInfo2 GUID that names it.
+#include <dvdmedia.h>
static QString wideToQString(LPCWSTR s)
{
return s ? QString::fromWCharArray(s) : QString();
}
+// Compressed formats worth forwarding untouched, best first. Same order and
+// the same FFmpeg codec names as kCompressedFormatPreference in
+// videodevice.cpp - that table is keyed by v4l2 pixel format, this one by the
+// FourCC DirectShow reports in BITMAPINFOHEADER::biCompression, but they mean
+// the same thing and should be kept in step.
+//
+// Cameras are inconsistent about which FourCC they use for the same stream,
+// so several map to one codec.
+static const struct
+{
+ const char *fourcc;
+ const char *ffmpeg_codec;
+} kDshowCompressedFormats[] =
+{
+ { "HEVC", "hevc" },
+ { "hevc", "hevc" },
+ { "H265", "hevc" },
+ { "h265", "hevc" },
+ { "H264", "h264" },
+ { "h264", "h264" },
+ { "X264", "h264" },
+ { "x264", "h264" },
+ { "AVC1", "h264" },
+ { "avc1", "h264" },
+ { "MJPG", "mjpeg" },
+ { "mjpg", "mjpeg" },
+ { "dmb1", "mjpeg" },
+ { "JPEG", "mjpeg" },
+};
+
+static QString fourccToString(DWORD fourcc)
+{
+ char c[5];
+ c[0] = (char)(fourcc & 0xff);
+ c[1] = (char)((fourcc >> 8) & 0xff);
+ c[2] = (char)((fourcc >> 16) & 0xff);
+ c[3] = (char)((fourcc >> 24) & 0xff);
+ c[4] = 0;
+
+ // Uncompressed types use small numbers rather than four characters -
+ // BI_RGB is 0 - which would otherwise come out as an unreadable string.
+ for (int i = 0; i < 4; i++)
+ {
+ if (c[i] < 0x20 || c[i] > 0x7e)
+ return QString("0x%1").arg((quint32)fourcc, 8, 16, QLatin1Char('0'));
+ }
+
+ return QString::fromLatin1(c);
+}
+
+// AM_MEDIA_TYPE owns two allocations of its own. DeleteMediaType() would do
+// this, but it lives in the DirectShow base classes rather than in any library
+// worth pulling in for one function.
+static void freeMediaType(AM_MEDIA_TYPE *mt)
+{
+ if (!mt)
+ return;
+
+ if (mt->cbFormat && mt->pbFormat)
+ CoTaskMemFree(mt->pbFormat);
+ if (mt->pUnk)
+ mt->pUnk->Release();
+
+ CoTaskMemFree(mt);
+}
+
+// Finds the camera behind one of the device strings enumeration handed out,
+// matching on either name for the same reason FFmpeg's demuxer does: the
+// friendly name for the common case, the device path where that was not
+// unique. Caller releases the moniker.
+static IMoniker *findCamera(const QString &device)
+{
+ ICreateDevEnum *devEnum = nullptr;
+ if (FAILED(CoCreateInstance(CLSID_SystemDeviceEnum, nullptr, CLSCTX_INPROC_SERVER,
+ IID_ICreateDevEnum, (void **)&devEnum)) || !devEnum)
+ return nullptr;
+
+ IEnumMoniker *enumMoniker = nullptr;
+ HRESULT hr = devEnum->CreateClassEnumerator(CLSID_VideoInputDeviceCategory, &enumMoniker, 0);
+ devEnum->Release();
+
+ if (hr != S_OK || !enumMoniker)
+ return nullptr;
+
+ IMoniker *moniker = nullptr;
+ IMoniker *found = nullptr;
+
+ while (!found && enumMoniker->Next(1, &moniker, nullptr) == S_OK)
+ {
+ IPropertyBag *bag = nullptr;
+ if (SUCCEEDED(moniker->BindToStorage(nullptr, nullptr, IID_IPropertyBag, (void **)&bag)) && bag)
+ {
+ VARIANT var;
+
+ VariantInit(&var);
+ if (SUCCEEDED(bag->Read(L"FriendlyName", &var, nullptr)) && var.vt == VT_BSTR &&
+ wideToQString(var.bstrVal) == device)
+ {
+ found = moniker;
+ }
+ VariantClear(&var);
+
+ if (!found)
+ {
+ VariantInit(&var);
+ if (SUCCEEDED(bag->Read(L"DevicePath", &var, nullptr)) && var.vt == VT_BSTR &&
+ wideToQString(var.bstrVal).replace(QLatin1Char(':'), QLatin1Char('_')) == device)
+ {
+ found = moniker;
+ }
+ VariantClear(&var);
+ }
+
+ bag->Release();
+ }
+
+ if (!found)
+ moniker->Release();
+ }
+
+ enumMoniker->Release();
+ return found;
+}
+
+// Which compressed formats a camera can hand over ready-made, best first.
+//
+// The counterpart of the v4l2 VIDIOC_ENUM_FMT walk in
+// VideoDevice::CompressedFormats(). DirectShow has no single call for it: the
+// camera has to be instantiated, its output pin found, and the pin's stream
+// capabilities enumerated one media type at a time.
+//
+// Note this instantiates the capture filter, which is also the step that fails
+// when Windows is refusing the app access to the camera - so an empty list
+// here can mean either "offers nothing compressed" or "cannot be opened at
+// all". Both lead to the same place, the caller simply transcodes instead.
+QStringList pica_dshow_compressed_formats(const QString &device)
+{
+ QStringList result;
+
+ if (device.isEmpty())
+ return result;
+
+ bool didInit = false;
+ HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
+ if (SUCCEEDED(hr))
+ didInit = true;
+ else if (hr != RPC_E_CHANGED_MODE)
+ return result;
+
+ IMoniker *moniker = findCamera(device);
+ if (!moniker)
+ {
+ if (didInit)
+ CoUninitialize();
+ return result;
+ }
+
+ IBaseFilter *filter = nullptr;
+ hr = moniker->BindToObject(nullptr, nullptr, IID_IBaseFilter, (void **)&filter);
+ moniker->Release();
+
+ if (FAILED(hr) || !filter)
+ {
+ qWarning() << "DirectShow: could not instantiate" << device
+ << "to ask what formats it offers";
+ if (didInit)
+ CoUninitialize();
+ return result;
+ }
+
+ QStringList offered;
+
+ IEnumPins *enumPins = nullptr;
+ if (SUCCEEDED(filter->EnumPins(&enumPins)) && enumPins)
+ {
+ IPin *pin = nullptr;
+
+ while (enumPins->Next(1, &pin, nullptr) == S_OK)
+ {
+ PIN_DIRECTION dir;
+ IAMStreamConfig *config = nullptr;
+
+ if (SUCCEEDED(pin->QueryDirection(&dir)) && dir == PINDIR_OUTPUT &&
+ SUCCEEDED(pin->QueryInterface(IID_IAMStreamConfig, (void **)&config)) && config)
+ {
+ int count = 0, size = 0;
+
+ if (SUCCEEDED(config->GetNumberOfCapabilities(&count, &size)) && size > 0)
+ {
+ QVector<quint8> caps(size);
+
+ for (int i = 0; i < count; i++)
+ {
+ AM_MEDIA_TYPE *mt = nullptr;
+ if (FAILED(config->GetStreamCaps(i, &mt, caps.data())) || !mt)
+ continue;
+
+ // Only the VIDEOINFOHEADER shapes carry a
+ // BITMAPINFOHEADER, which is where the FourCC lives.
+ const BITMAPINFOHEADER *bih = nullptr;
+
+ if (mt->formattype == FORMAT_VideoInfo &&
+ mt->cbFormat >= sizeof(VIDEOINFOHEADER))
+ bih = &((VIDEOINFOHEADER *)mt->pbFormat)->bmiHeader;
+ else if (mt->formattype == FORMAT_VideoInfo2 &&
+ mt->cbFormat >= sizeof(VIDEOINFOHEADER2))
+ bih = &((VIDEOINFOHEADER2 *)mt->pbFormat)->bmiHeader;
+
+ if (bih)
+ {
+ QString fourcc = fourccToString(bih->biCompression);
+ if (!offered.contains(fourcc))
+ offered << fourcc;
+ }
+
+ freeMediaType(mt);
+ }
+ }
+ }
+
+ if (config)
+ config->Release();
+ pin->Release();
+ }
+
+ enumPins->Release();
+ }
+
+ filter->Release();
+
+ // Worth logging whole, not just the ones we recognise. A camera offering
+ // nothing but uncompressed types here - NV12, YUY2 - while advertising
+ // MJPEG on the same hardware under v4l2 is the signature of the Windows
+ // Camera Frame Server, which mediates integrated cameras and hands
+ // DirectShow the already-decoded output. Nothing to be done about that
+ // from here, but it is worth being able to tell it apart from a FourCC we
+ // simply failed to map.
+ qDebug() << "DirectShow:" << device << "offers"
+ << (offered.isEmpty() ? QStringLiteral("(nothing)") : offered.join(QLatin1String(", ")));
+
+ // Report in our preference order rather than in the order the camera
+ // happened to list them.
+ for (unsigned int i = 0; i < sizeof(kDshowCompressedFormats) / sizeof(kDshowCompressedFormats[0]); i++)
+ {
+ QString codec = QLatin1String(kDshowCompressedFormats[i].ffmpeg_codec);
+
+ if (offered.contains(QLatin1String(kDshowCompressedFormats[i].fourcc)) && !result.contains(codec))
+ result << codec;
+ }
+
+ if (didInit)
+ CoUninitialize();
+
+ return result;
+}
+
// Lists the cameras DirectShow knows about.
//
// Capture itself goes through FFmpeg's "dshow" demuxer rather than through
generated by cgit v1.2.1 (git 2.18.0) at 2026-10-18 15:35:41 +0000