diff options
| author | Anton <anton@picapica.im> | 2026-10-04 20:27:09 +0400 |
|---|---|---|
| committer | Anton <anton@picapica.im> | 2026-10-04 20:27:09 +0400 |
| commit | 8f87c8491ffd9567df97979e9c18d4c148357870 (patch) | |
| tree | 3e8516066c96834defae2be933ce034d928ca7b3 | |
| parent | 7d6512f8051af51b89b6c9166c44607bfe1b83c3 (diff) | |
| download | pica-pica-8f87c8491ffd9567df97979e9c18d4c148357870.tar.gz pica-pica-8f87c8491ffd9567df97979e9c18d4c148357870.zip | |
pica-client: more hardware acceleration options on Windows
| -rw-r--r-- | configure.ac | 44 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 7 | ||||
| -rw-r--r-- | pica-client/callsettings.cpp | 25 | ||||
| -rw-r--r-- | pica-client/callsettings.h | 28 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 174 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.h | 25 | ||||
| -rw-r--r-- | pica-client/videodevice.cpp | 385 | ||||
| -rw-r--r-- | pica-client/videodevice.h | 48 | ||||
| -rwxr-xr-x | windows/cross_build_mxe.sh | 2 | ||||
| -rw-r--r-- | windows/mxe_ffmpeg_strip_excess_deps.patch | 8 |
10 files changed, 612 insertions, 134 deletions
diff --git a/configure.ac b/configure.ac index f7b48f6..2107786 100644 --- a/configure.ac +++ b/configure.ac @@ -92,15 +92,15 @@ AC_ARG_ENABLE([vaapi], AS_HELP_STRING([--disable-vaapi], [Disable VAAPI accelerated video encoding, decoding and rendering])) -#checking if Direct3D 12 hardware encoding is wanted. The Windows counterpart -#of the VAAPI encoding path above - there is no libva there, but FFmpeg can -#reach the same GPU encoders through D3D12 Video. Autodetected the same way, -#and only ever looked for when building for Windows. +#checking if Direct3D 12 hardware encoding and decoding is wanted. The Windows +#counterpart of the VAAPI path above - there is no libva there, but FFmpeg can +#reach the same GPU encoders and decoders through D3D12 Video. Autodetected the +#same way, and only ever looked for when building for Windows. AC_ARG_ENABLE([d3d12va], AS_HELP_STRING([--enable-d3d12va], - [Enable Direct3D 12 accelerated video encoding, Windows only (default: autodetect)]) + [Enable Direct3D 12 accelerated video encoding and decoding, Windows only (default: autodetect)]) AS_HELP_STRING([--disable-d3d12va], - [Disable Direct3D 12 accelerated video encoding])) + [Disable Direct3D 12 accelerated video encoding and decoding])) #checking if PulseAudio device enumeration is wanted. Only affects what the #settings dialog can list; see the check further down. @@ -324,19 +324,25 @@ AS_IF([test "x$PICA_VAAPI_ENABLED" = "xyes" && test "x$enable_vaapi_x11" != "xno ]) ]) -#check whether FFmpeg knows how to encode through Direct3D 12, which is how a -#Windows build reaches the GPU: libva is not part of that platform, but the -#hwcontext machinery the VAAPI path uses carries a D3D12 device type as well, -#and libavcodec has h264_d3d12va and hevc_d3d12va encoders to go with it. +#check whether FFmpeg knows how to encode and decode through Direct3D 12, one +#of the ways a Windows build reaches the GPU: libva is not part of that +#platform, but the hwcontext machinery the VAAPI path uses carries a D3D12 +#device type as well, and libavcodec has h264_d3d12va and hevc_d3d12va encoders +#and D3D12 hwaccels for its decoders to go with it. # #Nothing is linked for this - libavutil loads d3d12.dll itself when a device is #created - so all there is to check is that the headers are new enough to name -#the device type and the pixel format. Whether those encoders were compiled -#into the libavcodec being built against, and whether the machine the client -#ends up running on has a GPU that can encode, are both runtime questions, -#answered by falling back to the software encoder. +#the device type and the pixel format. Whether those encoders and hwaccels were +#compiled into the libavcodec being built against, and whether the machine the +#client ends up running on has a GPU that can use them, are both runtime +#questions, answered by falling back to software. +# +#The other Windows paths - Media Foundation encoding, Direct3D 11 and DXVA2 +#decoding - need no check of their own: FFmpeg's headers have named them for +#far longer than any FFmpeg this builds against, so they are compiled in on +#every Windows build and left to the runtime to find out whether they work. AS_IF([test "x$TARGET_OS" = "xwindows" && test "x$enable_d3d12va" != "xno"], [ - AC_MSG_CHECKING([whether FFmpeg supports Direct3D 12 video encoding]) + AC_MSG_CHECKING([whether FFmpeg supports Direct3D 12 video encoding and decoding]) pica_save_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $FFMPEG_CFLAGS" @@ -350,21 +356,21 @@ AS_IF([test "x$TARGET_OS" = "xwindows" && test "x$enable_d3d12va" != "xno"], [ (void)device; (void)format;]])], [ PICA_D3D12VA_ENABLED=yes - AC_DEFINE([HAVE_D3D12VA], [1], [Define to 1 if FFmpeg can encode video through Direct3D 12]) + AC_DEFINE([HAVE_D3D12VA], [1], [Define to 1 if FFmpeg can encode and decode video through Direct3D 12]) AC_MSG_RESULT([yes]) ], [ AC_MSG_RESULT([no]) AS_IF([test "x$enable_d3d12va" = "xyes"], - [AC_MSG_ERROR(["Direct3D 12 encoding was requested but this FFmpeg has no D3D12 hardware context"])], - [AC_MSG_WARN(["this FFmpeg has no D3D12 hardware context - building without Direct3D 12 encoding"])]) + [AC_MSG_ERROR(["Direct3D 12 acceleration was requested but this FFmpeg has no D3D12 hardware context"])], + [AC_MSG_WARN(["this FFmpeg has no D3D12 hardware context - building without Direct3D 12 acceleration"])]) ]) CPPFLAGS="$pica_save_CPPFLAGS" ]) AS_IF([test "x$TARGET_OS" != "xwindows" && test "x$enable_d3d12va" = "xyes"], - [AC_MSG_ERROR(["Direct3D 12 encoding was requested but it only exists on Windows"])]) + [AC_MSG_ERROR(["Direct3D 12 acceleration was requested but it only exists on Windows"])]) ]) AM_CONDITIONAL([PICA_VAAPI], [test "x$PICA_VAAPI_ENABLED" = "xyes"]) diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index 601f3e4..5b3c05c 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -271,7 +271,6 @@ void AudioVideoCallController::startVideoPipeline() cam->setMaxFragmentSize(PICA_MEDIA_SAFE_PAYLOAD); bool preferCompressed = st.loadValue("video.prefer_compressed", 0).toBool(); - bool vaapiEncoding = st.loadValue("video.vaapi_encoding", 0).toBool(); // Kept so that video_capture_started() can tell whether the camera // honoured it, which is not something the camera reports. @@ -287,7 +286,7 @@ void AudioVideoCallController::startVideoPipeline() Q_ARG(int, cs.captureFrameRate), Q_ARG(bool, preferCompressed), Q_ARG(QString, cs.videoCodec), Q_ARG(int, cs.videoBitrateKbps * 1000), - Q_ARG(bool, vaapiEncoding)); + Q_ARG(int, cs.videoEncodingAcceleration)); QMetaObject::invokeMethod(cam, "Capture", Qt::QueuedConnection); // remotevideo is configured lazily, once the peer's own 0x75 (see @@ -580,7 +579,7 @@ void AudioVideoCallController::incoming_video_params(QByteArray peer_id, QString return; Settings st(config_dbname); - bool vaapiDecoding = st.loadValue("video.vaapi_decoding", 0).toBool(); + const CallSettings cs = CallSettings::load(); bool vaapiRendering = st.loadValue("video.vaapi_rendering", 0).toBool(); #ifdef HAVE_VAAPI @@ -592,7 +591,7 @@ void AudioVideoCallController::incoming_video_params(QByteArray peer_id, QString QMetaObject::invokeMethod(remotevideo, "configurePlayback", Qt::QueuedConnection, Q_ARG(QString, codec), Q_ARG(int, (int)width), Q_ARG(int, (int)height), - Q_ARG(bool, vaapiDecoding), Q_ARG(bool, vaapiRendering)); + Q_ARG(int, cs.videoDecodingAcceleration), Q_ARG(bool, vaapiRendering)); QMetaObject::invokeMethod(remotevideo, "Play", Qt::QueuedConnection); } diff --git a/pica-client/callsettings.cpp b/pica-client/callsettings.cpp index b97d46a..586fbae 100644 --- a/pica-client/callsettings.cpp +++ b/pica-client/callsettings.cpp @@ -101,6 +101,31 @@ CallSettings CallSettings::load() cs.captureHeight = st.loadValue("video.capture_height", kDefaultCaptureHeight).toInt(); cs.captureFrameRate = st.loadValue("video.capture_framerate", kDefaultCaptureFrameRate).toInt(); + // Only the keys this build has checkboxes for: a database carried over + // from another platform must not switch on a path that cannot exist here. + cs.videoEncodingAcceleration = 0; + cs.videoDecodingAcceleration = 0; +#ifdef HAVE_VAAPI + if (st.loadValue("video.vaapi_encoding", 0).toBool()) + cs.videoEncodingAcceleration |= VideoAccelerationVaapi; + if (st.loadValue("video.vaapi_decoding", 0).toBool()) + cs.videoDecodingAcceleration |= VideoAccelerationVaapi; +#endif +#ifdef HAVE_D3D12VA + if (st.loadValue("video.d3d12va_encoding", 0).toBool()) + cs.videoEncodingAcceleration |= VideoAccelerationD3d12va; + if (st.loadValue("video.d3d12va_decoding", 0).toBool()) + cs.videoDecodingAcceleration |= VideoAccelerationD3d12va; +#endif +#ifdef Q_OS_WIN + if (st.loadValue("video.mediafoundation_encoding", 0).toBool()) + cs.videoEncodingAcceleration |= VideoAccelerationMediaFoundation; + if (st.loadValue("video.d3d11va_decoding", 0).toBool()) + cs.videoDecodingAcceleration |= VideoAccelerationD3d11va; + if (st.loadValue("video.dxva2_decoding", 0).toBool()) + cs.videoDecodingAcceleration |= VideoAccelerationDxva2; +#endif + // A stored value that makes no sense - a hand-edited database, or a // setting written by a build that offered something this one does not - // would otherwise be passed to the encoder and fail the call rather than diff --git a/pica-client/callsettings.h b/pica-client/callsettings.h index 65419e4..868b156 100644 --- a/pica-client/callsettings.h +++ b/pica-client/callsettings.h @@ -60,6 +60,28 @@ const int kDefaultCaptureWidth = 640; const int kDefaultCaptureHeight = 480; const int kDefaultCaptureFrameRate = 15; +// The hardware paths video may be encoded or decoded through, as flags to +// combine - one per checkbox on the "Video Devices" tab, and named after the +// FFmpeg component behind each. Which of them a build offers depends on the +// platform: VAAPI where there is libva, the rest on Windows only, and Direct3D +// 12 only with an FFmpeg new enough to have it (HAVE_D3D12VA). +// +// Each is a request rather than a guarantee. A path FFmpeg was built without, +// one with no usable device, or one that will not open for the stream at hand +// is passed over for the next one enabled, and with none left the work is +// done in software - see VideoDevice for the order they are tried in. +enum VideoAcceleration +{ + // Encoding and decoding. + VideoAccelerationVaapi = 0x01, + VideoAccelerationD3d12va = 0x02, + // Encoding only: FFmpeg has no Media Foundation decoders. + VideoAccelerationMediaFoundation = 0x04, + // Decoding only: FFmpeg has no Direct3D 11 or DXVA2 encoders. + VideoAccelerationD3d11va = 0x08, + VideoAccelerationDxva2 = 0x10 +}; + struct CallSettings { // FFmpeg codec names, e.g. "h264" and "opus". @@ -77,6 +99,12 @@ struct CallSettings int captureHeight; int captureFrameRate; + // VideoAcceleration flags for the two directions, out of the checkboxes + // this build offers. Rendering straight from GPU memory is not among them: + // it belongs to the window drawing the frames rather than to the decoder. + int videoEncodingAcceleration; + int videoDecodingAcceleration; + // Reads the stored settings, filling in the defaults above for anything // that was never set. Must be called from a thread that may touch the // settings database - not from a capture or playback loop, see diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index a768b55..8462a5b 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -26,6 +26,7 @@ #include <QGroupBox> #include <QVBoxLayout> #include <QHBoxLayout> +#include <QGridLayout> #include <QFormLayout> #include <QLabel> #include <QDebug> @@ -375,24 +376,72 @@ SettingsDialog::SettingsDialog(QWidget *parent) : cbPreferCompressed = new QCheckBox(tr("Prefer compressed formats if provided by the camera"), this); -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) - // Encoding is offered wherever there is an accelerator to encode on, - // named after the one this build uses; decoding and rendering below are - // VAAPI only, since the surfaces a D3D12 decoder produces have no way yet - // to reach the widget that would draw them. #ifdef HAVE_VAAPI cbVaapiEncoding = new QCheckBox(tr("enable VAAPI-accelerated hardware encoding"), this); -#else - cbVaapiEncoding = new QCheckBox(tr("enable Direct3D 12 accelerated hardware encoding"), this); -#endif -#endif - -#ifdef HAVE_VAAPI cbVaapiDecoding = new QCheckBox(tr("enable VAAPI-accelerated hardware decoding"), this); cbVaapiRendering = new QCheckBox(tr("enable VAAPI accelerated hardware rendering"), this); // Drawing a VA surface means having decoded into one first, so rendering // brings decoding along with it whether or not it was asked for. cbVaapiRendering->setToolTip(tr("Requires hardware decoding, which is enabled along with it")); + offerAcceleration(cbVaapiEncoding, VideoAccelerationVaapi, true); + offerAcceleration(cbVaapiDecoding, VideoAccelerationVaapi, false); +#endif + +#ifdef Q_OS_WIN + // Encoding and decoding side by side rather than one checkbox per line: + // five lines of them would push the preview below off a small screen. + QGroupBox *gbHwAccel = new QGroupBox(tr("GPU acceleration"), this); + QGridLayout *hwAccelLayout = new QGridLayout(); + int encodingRow = 1; + int decodingRow = 1; + + hwAccelLayout->addWidget(new QLabel(tr("Encoding:"), this), 0, 0); + hwAccelLayout->addWidget(new QLabel(tr("Decoding:"), this), 0, 1); + + // Listed in the order VideoDevice tries them, which is what decides the + // one used when several are ticked. +#ifdef HAVE_D3D12VA + cbD3d12vaEncoding = new QCheckBox(tr("Direct3D 12"), this); + cbD3d12vaEncoding->setToolTip(tr("Encodes on the GPU through Direct3D 12 Video. Needs a " + "recent Windows and a GPU driver that implements Direct3D " + "12 video encoding.\n" + "Tried before Media Foundation when both are ticked.")); + offerAcceleration(cbD3d12vaEncoding, VideoAccelerationD3d12va, true); + hwAccelLayout->addWidget(cbD3d12vaEncoding, encodingRow++, 0); +#endif + + cbMediaFoundationEncoding = new QCheckBox(tr("Media Foundation"), this); + cbMediaFoundationEncoding->setToolTip(tr("Encodes on the GPU through the hardware encoder " + "the GPU driver registers with Windows. Works with " + "older GPUs and drivers too. Microsoft's software " + "encoder is never used through this - without a " + "hardware one, encoding falls back to libx264.")); + offerAcceleration(cbMediaFoundationEncoding, VideoAccelerationMediaFoundation, true); + hwAccelLayout->addWidget(cbMediaFoundationEncoding, encodingRow++, 0); + + cbD3d11vaDecoding = new QCheckBox(tr("Direct3D 11"), this); + cbD3d11vaDecoding->setToolTip(tr("Decodes on the GPU through Direct3D 11 Video, the most " + "widely supported way on Windows 8 and later.\n" + "Tried first when several are ticked.")); + offerAcceleration(cbD3d11vaDecoding, VideoAccelerationD3d11va, false); + hwAccelLayout->addWidget(cbD3d11vaDecoding, decodingRow++, 1); + +#ifdef HAVE_D3D12VA + cbD3d12vaDecoding = new QCheckBox(tr("Direct3D 12"), this); + cbD3d12vaDecoding->setToolTip(tr("Decodes on the GPU through Direct3D 12 Video.\n" + "Tried after Direct3D 11 when both are ticked.")); + offerAcceleration(cbD3d12vaDecoding, VideoAccelerationD3d12va, false); + hwAccelLayout->addWidget(cbD3d12vaDecoding, decodingRow++, 1); +#endif + + cbDxva2Decoding = new QCheckBox(tr("DXVA2"), this); + cbDxva2Decoding->setToolTip(tr("Decodes on the GPU through DirectX Video Acceleration 2, " + "on top of Direct3D 9 - for GPUs and drivers too old for " + "the others.\nTried last when several are ticked.")); + offerAcceleration(cbDxva2Decoding, VideoAccelerationDxva2, false); + hwAccelLayout->addWidget(cbDxva2Decoding, decodingRow++, 1); + + gbHwAccel->setLayout(hwAccelLayout); #endif btVideoTest = new QPushButton(tr("Test 📷"), this); @@ -429,13 +478,14 @@ SettingsDialog::SettingsDialog(QWidget *parent) : videodevLayout->addWidget(videoDevRefresh); videodevLayout->addLayout(videoFormatLayout); videodevLayout->addWidget(cbPreferCompressed); -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) - videodevLayout->addWidget(cbVaapiEncoding); -#endif #ifdef HAVE_VAAPI + videodevLayout->addWidget(cbVaapiEncoding); videodevLayout->addWidget(cbVaapiDecoding); videodevLayout->addWidget(cbVaapiRendering); #endif +#ifdef Q_OS_WIN + videodevLayout->addWidget(gbHwAccel); +#endif videodevLayout->addWidget(btVideoTest); // The preview takes the tab's spare height, where it used to go to a spacer // at the bottom. Only one of the two preview widgets is ever visible and a @@ -896,11 +946,6 @@ void SettingsDialog::toggleVideoTest() videoFpsElapsed.start(); videoFpsTimer->start(); - bool vaapiEncoding = false; -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) - vaapiEncoding = cbVaapiEncoding->isChecked(); -#endif - videoTestPathReport.clear(); // The settings as they stand in the dialog, not as they were last stored: @@ -913,7 +958,7 @@ void SettingsDialog::toggleVideoTest() Q_ARG(bool, cbPreferCompressed->isChecked()), Q_ARG(QString, callVideoCodec->itemData(callVideoCodec->currentIndex()).toString()), Q_ARG(int, callVideoBitrate->value() * 1000), - Q_ARG(bool, vaapiEncoding)); + Q_ARG(int, selectedVideoEncodingAcceleration())); QMetaObject::invokeMethod(testCam, "Capture", Qt::QueuedConnection); // The decoder is only started once the camera has settled on a format, @@ -942,10 +987,8 @@ void SettingsDialog::videoTestCaptureStarted(QString codec, int width, int heigh : tr("%1 %2x%3").arg(codec).arg(width).arg(height); videoPreview->setText(tr("Capturing %1...").arg(videoTestFormat)); - bool vaapiDecoding = false; bool vaapiRendering = false; #ifdef HAVE_VAAPI - vaapiDecoding = cbVaapiDecoding->isChecked(); // Asking for frames to be kept in GPU memory once it is known that they // cannot be drawn from there would only mean starting each test with a // blank preview until it failed again. @@ -955,7 +998,8 @@ void SettingsDialog::videoTestCaptureStarted(QString codec, int width, int heigh QMetaObject::invokeMethod(testDecoder, "configurePlayback", Qt::QueuedConnection, Q_ARG(QString, codec), Q_ARG(int, width), Q_ARG(int, height), - Q_ARG(bool, vaapiDecoding), Q_ARG(bool, vaapiRendering)); + Q_ARG(int, selectedVideoDecodingAcceleration()), + Q_ARG(bool, vaapiRendering)); QMetaObject::invokeMethod(testDecoder, "Play", Qt::QueuedConnection); } @@ -1413,6 +1457,64 @@ int SettingsDialog::selectedVideoFrameRate() const return (ok && fps > 0) ? fps : kDefaultCaptureFrameRate; } +void SettingsDialog::offerAcceleration(QCheckBox *cb, VideoAcceleration acceleration, bool encoding) +{ + if (VideoDevice::AccelerationBuiltIn(acceleration, encoding)) + return; + + cb->setEnabled(false); + cb->setToolTip(encoding + ? tr("Not available: the FFmpeg this program runs with was built without " + "these encoders.") + : tr("Not available: the FFmpeg this program runs with was built without " + "support for these decoders.")); +} + +int SettingsDialog::selectedVideoEncodingAcceleration() const +{ + int flags = 0; + + // isEnabled() as well, so that a path greyed out by offerAcceleration() + // but ticked in a stored setting is not passed on - it would only be + // skipped with a warning further down. +#ifdef HAVE_VAAPI + if (cbVaapiEncoding->isEnabled() && cbVaapiEncoding->isChecked()) + flags |= VideoAccelerationVaapi; +#endif +#ifdef HAVE_D3D12VA + if (cbD3d12vaEncoding->isEnabled() && cbD3d12vaEncoding->isChecked()) + flags |= VideoAccelerationD3d12va; +#endif +#ifdef Q_OS_WIN + if (cbMediaFoundationEncoding->isEnabled() && cbMediaFoundationEncoding->isChecked()) + flags |= VideoAccelerationMediaFoundation; +#endif + + return flags; +} + +int SettingsDialog::selectedVideoDecodingAcceleration() const +{ + int flags = 0; + +#ifdef HAVE_VAAPI + if (cbVaapiDecoding->isEnabled() && cbVaapiDecoding->isChecked()) + flags |= VideoAccelerationVaapi; +#endif +#ifdef HAVE_D3D12VA + if (cbD3d12vaDecoding->isEnabled() && cbD3d12vaDecoding->isChecked()) + flags |= VideoAccelerationD3d12va; +#endif +#ifdef Q_OS_WIN + if (cbD3d11vaDecoding->isEnabled() && cbD3d11vaDecoding->isChecked()) + flags |= VideoAccelerationD3d11va; + if (cbDxva2Decoding->isEnabled() && cbDxva2Decoding->isChecked()) + flags |= VideoAccelerationDxva2; +#endif + + return flags; +} + void SettingsDialog::callAudioCodecChanged() { const QString codec = callAudioCodec->itemData(callAudioCodec->currentIndex()).toString(); @@ -1660,13 +1762,20 @@ void SettingsDialog::loadSettings() // actually changes - and the stored codec is often the one already showing. callAudioCodecChanged(); -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) - cbVaapiEncoding->setChecked(st.loadValue("video.vaapi_encoding", 0).toBool()); -#endif #ifdef HAVE_VAAPI + cbVaapiEncoding->setChecked(st.loadValue("video.vaapi_encoding", 0).toBool()); cbVaapiDecoding->setChecked(st.loadValue("video.vaapi_decoding", 0).toBool()); cbVaapiRendering->setChecked(st.loadValue("video.vaapi_rendering", 0).toBool()); #endif +#ifdef HAVE_D3D12VA + cbD3d12vaEncoding->setChecked(st.loadValue("video.d3d12va_encoding", 0).toBool()); + cbD3d12vaDecoding->setChecked(st.loadValue("video.d3d12va_decoding", 0).toBool()); +#endif +#ifdef Q_OS_WIN + cbMediaFoundationEncoding->setChecked(st.loadValue("video.mediafoundation_encoding", 0).toBool()); + cbD3d11vaDecoding->setChecked(st.loadValue("video.d3d11va_decoding", 0).toBool()); + cbDxva2Decoding->setChecked(st.loadValue("video.dxva2_decoding", 0).toBool()); +#endif QString audioCapDevVal = st.loadValue("audio.capture_device", "default").toString(); int audioCapDevItem = audioCaptureDev->findData(audioCapDevVal); @@ -1759,13 +1868,20 @@ void SettingsDialog::storeSettings() audioBitrateKbps = callAudioBitrate->value(); st.storeValue("call.audio_bitrate_kbps", QString::number(audioBitrateKbps)); -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) - st.storeValue("video.vaapi_encoding", cbVaapiEncoding->isChecked() ? "1" : "0"); -#endif #ifdef HAVE_VAAPI + st.storeValue("video.vaapi_encoding", cbVaapiEncoding->isChecked() ? "1" : "0"); st.storeValue("video.vaapi_decoding", cbVaapiDecoding->isChecked() ? "1" : "0"); st.storeValue("video.vaapi_rendering", cbVaapiRendering->isChecked() ? "1" : "0"); #endif +#ifdef HAVE_D3D12VA + st.storeValue("video.d3d12va_encoding", cbD3d12vaEncoding->isChecked() ? "1" : "0"); + st.storeValue("video.d3d12va_decoding", cbD3d12vaDecoding->isChecked() ? "1" : "0"); +#endif +#ifdef Q_OS_WIN + st.storeValue("video.mediafoundation_encoding", cbMediaFoundationEncoding->isChecked() ? "1" : "0"); + st.storeValue("video.d3d11va_decoding", cbD3d11vaDecoding->isChecked() ? "1" : "0"); + st.storeValue("video.dxva2_decoding", cbDxva2Decoding->isChecked() ? "1" : "0"); +#endif st.storeValue("audio.capture_device", audioCaptureDev->itemData(audioCaptureDev->currentIndex()).toString()); st.storeValue("audio.playback_device", audioPlaybackDev->itemData(audioPlaybackDev->currentIndex()).toString()); st.storeValue("audio.ring_device", audioRingDev->itemData(audioRingDev->currentIndex()).toString()); diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h index 357edac..50b90c8 100644 --- a/pica-client/dialogs/settingsdialog.h +++ b/pica-client/dialogs/settingsdialog.h @@ -199,6 +199,31 @@ private: // from GPU memory; only one of the two is ever visible. VaapiRenderWidget *videoPreviewGpu; #endif + // One checkbox per Windows GPU path, encoding and decoding separately, + // since which of them works depends on the GPU and its driver and is only + // found out by trying - see VideoAcceleration. When several are ticked + // VideoDevice tries them in a fixed order, so each one can also be tested + // on its own by unticking the others. +#ifdef HAVE_D3D12VA + QCheckBox *cbD3d12vaEncoding; + QCheckBox *cbD3d12vaDecoding; +#endif +#ifdef Q_OS_WIN + QCheckBox *cbMediaFoundationEncoding; + QCheckBox *cbD3d11vaDecoding; + QCheckBox *cbDxva2Decoding; +#endif + // Greys out a checkbox for a path the FFmpeg the client runs with was + // built without, saying so in its tooltip, rather than letting it be + // ticked and then quietly fall back to software. + void offerAcceleration(QCheckBox *cb, VideoAcceleration acceleration, bool encoding); + + // VideoAcceleration flags out of the checkboxes as they stand, for the + // local pipeline test - which runs on what the dialog shows rather than + // on what was last stored. + int selectedVideoEncodingAcceleration() const; + int selectedVideoDecodingAcceleration() const; + VideoDevice *testCam; VideoDevice *testDecoder; QThread testCamThread; diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp index 419d1df..e393ba6 100644 --- a/pica-client/videodevice.cpp +++ b/pica-client/videodevice.cpp @@ -57,9 +57,8 @@ extern "C" { #include <libavutil/mem.h> #include <libavutil/error.h> #include <libswscale/swscale.h> -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) #include <libavutil/hwcontext.h> -#endif +#include <libavutil/pixdesc.h> } // Largest amount of encoded data one 0x77 message can carry over a c2c or @@ -113,26 +112,55 @@ static const struct // offers, on the GPU and in software. Several names per codec because FFmpeg // names its external library encoders after the library rather than after the // codec, and names a hardware one after the API that drives it - VAAPI where -// there is libva, Direct3D 12 on Windows. The id as well, so that a build -// without the library named here can still fall back to whatever other encoder -// it has for the same codec. +// there is libva, Direct3D 12 and Media Foundation on Windows. The id as well, +// so that a build without the library named here can still fall back to +// whatever other encoder it has for the same codec. // -// A null hardware name means that API has no encoder for the codec: D3D12 -// Video defines H.264, HEVC and AV1 encoding and no VP9, so a call set to VP9 -// encodes in software on Windows however the acceleration setting stands. +// A null hardware name means that API has no encoder for the codec: neither +// D3D12 Video nor FFmpeg's Media Foundation wrapper has VP9 encoding, so a call +// set to VP9 encodes in software on Windows however the acceleration settings +// stand. static const struct { const char *codec; AVCodecID id; const char *vaapiEncoder; const char *d3d12vaEncoder; + const char *mediaFoundationEncoder; const char *swEncoder; } kVideoEncoders[] = { - { "h264", AV_CODEC_ID_H264, "h264_vaapi", "h264_d3d12va", "libx264" }, - { "hevc", AV_CODEC_ID_HEVC, "hevc_vaapi", "hevc_d3d12va", "libx265" }, - { "vp9", AV_CODEC_ID_VP9, "vp9_vaapi", nullptr, "libvpx-vp9" } + { "h264", AV_CODEC_ID_H264, "h264_vaapi", "h264_d3d12va", "h264_mf", "libx264" }, + { "hevc", AV_CODEC_ID_HEVC, "hevc_vaapi", "hevc_d3d12va", "hevc_mf", "libx265" }, + { "vp9", AV_CODEC_ID_VP9, "vp9_vaapi", nullptr, nullptr, "libvpx-vp9" } +}; + +#ifdef Q_OS_WIN +// The Windows GPU decoders, in the order they are tried when more than one is +// enabled. Direct3D 11 first: it is the one FFmpeg's own tools default to on +// Windows and the most exercised of the three. Direct3D 12 next, being newer +// to FFmpeg, and DXVA2 - Direct3D 9 underneath - last, for GPUs and drivers +// too old for either of the others. +// +// The pixel format is the one the decoder hands back GPU frames in when driven +// through a device of the given type - D3D11 rather than the older +// D3D11VA_VLD, which is the same API set up by hand instead of through a +// hardware context. +static const struct +{ + VideoAcceleration flag; + AVHWDeviceType deviceType; + AVPixelFormat pixelFormat; + const char *name; +} kWindowsHwDecoders[] = +{ + { VideoAccelerationD3d11va, AV_HWDEVICE_TYPE_D3D11VA, AV_PIX_FMT_D3D11, "Direct3D 11" }, +#ifdef HAVE_D3D12VA + { VideoAccelerationD3d12va, AV_HWDEVICE_TYPE_D3D12VA, AV_PIX_FMT_D3D12, "Direct3D 12" }, +#endif + { VideoAccelerationDxva2, AV_HWDEVICE_TYPE_DXVA2, AV_PIX_FMT_DXVA2_VLD, "DXVA2" } }; +#endif static int videoEncoderIndex(const QString &codec) { @@ -241,25 +269,37 @@ static bool captureFormatLessThan(const VideoCaptureFormat &a, const VideoCaptur } #endif -#ifdef HAVE_VAAPI -// Picks the GPU pixel format out of what the decoder offers. Returning the -// first entry instead - which is what happens when VAAPI is not among them - -// leaves the decoder working in software, which is the wanted fallback. -static AVPixelFormat vaapi_get_format(AVCodecContext *ctx, const AVPixelFormat *formats) +// Picks the GPU pixel format out of what the decoder offers - the one the +// device attached to the decoder produces, which Play() leaves in +// ctx->opaque. When it is not among them, the first software format is +// returned instead, which leaves the decoder working on the CPU. +// +// Software rather than simply the first entry: the list starts with every +// hwaccel the FFmpeg build has, and one other than ours would only be tried, +// found not to match the device, and taken out of the list again. The same +// happens to ours when the GPU turns out not to support the stream after all - +// FFmpeg then asks again without it, which is when the warning below is +// printed. +static AVPixelFormat hw_get_format(AVCodecContext *ctx, const AVPixelFormat *formats) { - Q_UNUSED(ctx); + const AVPixelFormat wanted = (AVPixelFormat)(intptr_t)ctx->opaque; + const AVPixelFormat *software = nullptr; for (const AVPixelFormat *p = formats; *p != AV_PIX_FMT_NONE; p++) { - if (*p == AV_PIX_FMT_VAAPI) + if (*p == wanted) return *p; + + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(*p); + + if (!software && desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) + software = p; } qWarning() << "The GPU cannot decode this stream, decoding in software"; - return formats[0]; + return software ? *software : formats[0]; } -#endif QStringList VideoDevice::CompressedFormats(const QString &device) { @@ -669,7 +709,7 @@ VideoDevice::VideoDevice(QObject *parent) : QObject(parent), m_width(kDefaultCaptureWidth), m_height(kDefaultCaptureHeight), m_frameRate(kDefaultCaptureFrameRate), m_bitrate(kDefaultVideoBitrateKbps * 1000), m_preferCompressed(false), - m_useVaapi(false), m_useVaapiRender(false), m_abort(0), + m_acceleration(0), m_useVaapiRender(false), m_abort(0), m_maxFragmentSize(kMaxFragmentSize), m_displaySize(0), m_decoderStarted(0) { } @@ -718,7 +758,7 @@ VideoDevice::~VideoDevice() void VideoDevice::configureCapture(QString deviceName, int width, int height, int frameRate, bool preferCompressed, QString codec, int bitrate, - bool useVaapi) + int acceleration) { m_deviceName = deviceName; m_width = width; @@ -727,18 +767,18 @@ void VideoDevice::configureCapture(QString deviceName, int width, int height, in m_codec = codec; m_bitrate = bitrate > 0 ? bitrate : kDefaultVideoBitrateKbps * 1000; m_preferCompressed = preferCompressed; - m_useVaapi = useVaapi; + m_acceleration = acceleration; } void VideoDevice::configurePlayback(QString codec, int width, int height, - bool useVaapi, bool useVaapiRender) + int acceleration, bool useVaapiRender) { m_codec = codec; m_width = width; m_height = height; // Drawing a VA surface requires having decoded into one in the first - // place, so asking for GPU rendering turns on GPU decoding regardless. - m_useVaapi = useVaapi || useVaapiRender; + // place, so asking for GPU rendering turns on VAAPI decoding regardless. + m_acceleration = acceleration | (useVaapiRender ? VideoAccelerationVaapi : 0); m_useVaapiRender = useVaapiRender; // A renderer that failed during an earlier call says nothing about this // one - it may well be a different window. @@ -1297,13 +1337,16 @@ static AVPixelFormat swscaleSourceFormat(AVPixelFormat format, bool *fullRange) // live in - from the pool onwards it is the same hwcontext machinery, and the // encoder is configured exactly as the software one is below. // +// options are the encoder's private options, as avcodec_open2() takes them; +// null for none. +// // Returns the opened context with the pool it draws from in *hw_frames_ref, or // null with nothing left allocated and the reason logged, which leaves the // caller to encode in software. static AVCodecContext *openHwEncoder(const AVCodec *enc, AVBufferRef *hw_device_ref, AVPixelFormat hw_pix_fmt, int width, int height, AVRational frameRate, int64_t bitrate, int gopSize, - AVBufferRef **hw_frames_ref) + AVDictionary **options, AVBufferRef **hw_frames_ref) { AVCodecContext *enc_ctx = avcodec_alloc_context3(enc); @@ -1350,7 +1393,7 @@ static AVCodecContext *openHwEncoder(const AVCodec *enc, AVBufferRef *hw_device_ // No preset/tune here: those belong to the software encoders and mean // nothing to a hardware one. - ret = avcodec_open2(enc_ctx, enc, nullptr); + ret = avcodec_open2(enc_ctx, enc, options); if (ret < 0) { @@ -1365,6 +1408,64 @@ static AVCodecContext *openHwEncoder(const AVCodec *enc, AVBufferRef *hw_device_ } #endif +#ifdef Q_OS_WIN +// Opens one of FFmpeg's Media Foundation encoders. These reach the GPU through +// the encoder the vendor's driver registers with Windows rather than through a +// device of our own, and take their input in system memory, so unlike +// openHwEncoder() there is no device and no surface pool here: NV12 frames go +// in as they are and the driver does the upload. +// +// Returns the opened context, or null with the reason logged. +static AVCodecContext *openMediaFoundationEncoder(const AVCodec *enc, int width, int height, + AVRational frameRate, int64_t bitrate, + int gopSize) +{ + AVCodecContext *enc_ctx = avcodec_alloc_context3(enc); + + if (!enc_ctx) + return nullptr; + + enc_ctx->width = width; + enc_ctx->height = height; + // What the hardware encoders take natively; YUV420P would be converted + // inside the encoder instead of by the scaler we run anyway. + enc_ctx->pix_fmt = AV_PIX_FMT_NV12; + enc_ctx->time_base = av_inv_q(frameRate); + enc_ctx->framerate = frameRate; + enc_ctx->bit_rate = bitrate; + enc_ctx->gop_size = gopSize; + enc_ctx->max_b_frames = 0; + // Becomes CODECAPI_AVLowLatencyMode, which stops the encoder holding + // frames back to look ahead. + enc_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY; + + // Hardware only. Without it FFmpeg would also accept Microsoft's own + // software H.264 encoder, which is worse at the job than the libx264 this + // falls back to when no hardware encoder opens - so "GPU encoding" in the + // status line can be taken at its word. + av_opt_set_int(enc_ctx->priv_data, "hw_encoding", 1, 0); + // Tells the driver what the stream is for, the nearest Media Foundation + // has to x264's zerolatency tune. Constant bitrate because a call has a + // fixed amount of network to fit into. Both are hints that a driver is free + // to ignore, and FFmpeg does not treat them failing as an error. + av_opt_set(enc_ctx->priv_data, "scenario", "video_conference", 0); + av_opt_set(enc_ctx->priv_data, "rate_control", "cbr", 0); + + int ret = avcodec_open2(enc_ctx, enc, nullptr); + + if (ret < 0) + { + qWarning() << QString("The %1 encoder would not open (%2) - no hardware Media Foundation " + "encoder for this codec, or it refused the settings") + .arg(QLatin1String(enc->name), ff_errstr(ret)); + avcodec_free_context(&enc_ctx); + return nullptr; + } + + return enc_ctx; +} +#endif + void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) { const AVRational frameRate = cameraFrameRate(in_st, m_frameRate); @@ -1408,7 +1509,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) AVBufferRef *hw_frames_ref = nullptr; #ifdef HAVE_VAAPI - if (m_useVaapi && VaapiContext::isAvailable()) + if ((m_acceleration & VideoAccelerationVaapi) && VaapiContext::isAvailable()) { enc = avcodec_find_encoder_by_name(kVideoEncoders[encIdx].vaapiEncoder); AVBufferRef *hw_device_ref = enc ? VaapiContext::deviceRef() : nullptr; @@ -1417,7 +1518,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) { enc_ctx = openHwEncoder(enc, hw_device_ref, AV_PIX_FMT_VAAPI, m_width, m_height, frameRate, m_bitrate, gopSize, - &hw_frames_ref); + nullptr, &hw_frames_ref); hardware = enc_ctx != nullptr; av_buffer_unref(&hw_device_ref); @@ -1445,9 +1546,9 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) // gets a hardware encoder if only one of the two comes up. // // The device is created per capture session and dropped with it, unlike - // the VAAPI one: nothing else in the client holds a D3D12 device, since - // decoding and rendering do not go through it. - if (!hardware && m_useVaapi) + // the VAAPI one: nothing else in the client holds a D3D12 device - the + // decoder makes its own, and rendering does not go through one. + if (!hardware && (m_acceleration & VideoAccelerationD3d12va)) { const char *encoderName = kVideoEncoders[encIdx].d3d12vaEncoder; @@ -1460,11 +1561,20 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) if (hw_device_ref) { + // The D3D12 encoders keep up to async_depth frames in flight and + // hand a packet back only once that many have gone in, so the + // default of two holds every frame back by one - a frame interval + // of latency for nothing, with one frame at a time coming in. + AVDictionary *options = nullptr; + + av_dict_set(&options, "async_depth", "1", 0); + enc_ctx = openHwEncoder(enc, hw_device_ref, AV_PIX_FMT_D3D12, m_width, m_height, frameRate, m_bitrate, gopSize, - &hw_frames_ref); + &options, &hw_frames_ref); hardware = enc_ctx != nullptr; + av_dict_free(&options); av_buffer_unref(&hw_device_ref); } else if (enc) @@ -1492,6 +1602,39 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) } #endif +#ifdef Q_OS_WIN + // Last of the hardware paths: Media Foundation goes through whatever + // encoder the GPU vendor's driver has registered with Windows, so it also + // covers GPUs, drivers and versions of Windows that predate D3D12 video + // encoding. + if (!hardware && (m_acceleration & VideoAccelerationMediaFoundation)) + { + const char *encoderName = kVideoEncoders[encIdx].mediaFoundationEncoder; + + enc = encoderName ? avcodec_find_encoder_by_name(encoderName) : nullptr; + + if (enc) + { + enc_ctx = openMediaFoundationEncoder(enc, m_width, m_height, frameRate, + m_bitrate, gopSize); + hardware = enc_ctx != nullptr; + } + else if (encoderName) + { + qWarning() << QString("FFmpeg has no %1 encoder, encoding in software") + .arg(QLatin1String(encoderName)); + } + else + { + qWarning() << QString("Media Foundation cannot encode %1, encoding in software") + .arg(QLatin1String(kVideoEncoders[encIdx].codec)); + } + + if (!hardware) + enc = nullptr; + } +#endif + // Named rather than taken from enc->name because the VAAPI branch above // may have left a different encoder in enc; filled in below for the // software path, which is the only one the private options apply to. @@ -1571,11 +1714,13 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) emit captureStarted(QLatin1String(avcodec_get_name(enc_ctx->codec_id)), enc_ctx->width, enc_ctx->height, av_q2d(frameRate)); - // What the camera's frames are scaled into. With a hardware encoder this - // is only a staging buffer in system memory - NV12, which is what the - // GPU encoders take - that gets uploaded to a GPU surface below; the - // encoder itself consumes surfaces and never sees this frame. - AVPixelFormat sw_pix_fmt = hardware ? AV_PIX_FMT_NV12 : enc_ctx->pix_fmt; + // What the camera's frames are scaled into. With an encoder that draws on + // a surface pool this is only a staging buffer in system memory - NV12, + // which is what the GPU encoders take - that gets uploaded to a GPU + // surface below; the encoder itself consumes surfaces and never sees this + // frame. Every other encoder, Media Foundation's included, takes it as it + // is. + AVPixelFormat sw_pix_fmt = enc_ctx->hw_frames_ctx ? AV_PIX_FMT_NV12 : enc_ctx->pix_fmt; AVFrame *enc_frame = av_frame_alloc(); if (enc_frame) @@ -1675,13 +1820,13 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) enc_frame->pts = pts++; - // What actually goes to the encoder: the staging frame itself in - // software, or a GPU surface holding a copy of it in hardware. + // What actually goes to the encoder: the staging frame itself, or + // a GPU surface holding a copy of it for an encoder that draws on + // a surface pool. AVFrame *frame_to_encode = enc_frame; -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) AVFrame *hw_frame = nullptr; - if (hardware) + if (enc_ctx->hw_frames_ctx) { hw_frame = av_frame_alloc(); @@ -1701,13 +1846,26 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) hw_frame->pts = enc_frame->pts; frame_to_encode = hw_frame; } -#endif int send_ret = avcodec_send_frame(enc_ctx, frame_to_encode); -#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA) + + // An encoder with an input queue of its own - Media Foundation's + // hardware ones are - can refuse a frame until the packets it has + // ready are taken. Take them and offer the frame again, rather than + // dropping it and reporting a failure that is not one. + if (send_ret == AVERROR(EAGAIN)) + { + while (avcodec_receive_packet(enc_ctx, out_pkt) == 0) + { + emitFragments(out_pkt->data, out_pkt->size); + av_packet_unref(out_pkt); + } + + send_ret = avcodec_send_frame(enc_ctx, frame_to_encode); + } + if (hw_frame) av_frame_free(&hw_frame); -#endif if (send_ret != 0) { @@ -1755,12 +1913,16 @@ void VideoDevice::Play() dec = nullptr; AVCodecContext *dec_ctx = dec ? avcodec_alloc_context3(dec) : nullptr; + // The GPU API the decoder ended up attached to, for the status line; null + // while decoding in software. + const char *hwDecoderName = nullptr; + #ifdef HAVE_VAAPI // Has to be attached before opening the decoder. Unlike encoding, there is // no separate hardware decoder to look up: the ordinary decoder is told to // use a VAAPI device, and get_format then picks the GPU pixel format when // the codec and hardware between them can manage it. - if (m_useVaapi && dec_ctx && VaapiContext::isAvailable()) + if ((m_acceleration & VideoAccelerationVaapi) && dec_ctx && VaapiContext::isAvailable()) { AVBufferRef *hw_device_ref = VaapiContext::deviceRef(); @@ -1768,7 +1930,9 @@ void VideoDevice::Play() { // The context takes ownership of this reference. dec_ctx->hw_device_ctx = hw_device_ref; - dec_ctx->get_format = vaapi_get_format; + dec_ctx->opaque = (void *)(intptr_t)AV_PIX_FMT_VAAPI; + dec_ctx->get_format = hw_get_format; + hwDecoderName = "VAAPI"; } else { @@ -1777,6 +1941,36 @@ void VideoDevice::Play() } #endif +#ifdef Q_OS_WIN + // The same arrangement on Windows, with a device of our own per decode + // session - created here and freed with the decoder - since nothing else in + // the client holds one. The first API in kWindowsHwDecoders whose device + // comes up is the one used. + for (unsigned int i = 0; + dec_ctx && !dec_ctx->hw_device_ctx && i < sizeof(kWindowsHwDecoders) / sizeof(kWindowsHwDecoders[0]); + i++) + { + if (!(m_acceleration & kWindowsHwDecoders[i].flag)) + continue; + + AVBufferRef *hw_device_ref = nullptr; + int ret = av_hwdevice_ctx_create(&hw_device_ref, kWindowsHwDecoders[i].deviceType, + nullptr, nullptr, 0); + + if (ret < 0) + { + qWarning() << QString("No usable %1 decoding device (%2)") + .arg(QLatin1String(kWindowsHwDecoders[i].name), ff_errstr(ret)); + continue; + } + + dec_ctx->hw_device_ctx = hw_device_ref; + dec_ctx->opaque = (void *)(intptr_t)kWindowsHwDecoders[i].pixelFormat; + dec_ctx->get_format = hw_get_format; + hwDecoderName = kWindowsHwDecoders[i].name; + } +#endif + if (!dec || !dec_ctx || avcodec_open2(dec_ctx, dec, nullptr) < 0) { QString msg = QString("Unsupported call video codec: %1").arg(m_codec); @@ -1847,15 +2041,16 @@ void VideoDevice::Play() while (avcodec_receive_frame(dec_ctx, dec_frame) == 0) { - bool decodedOnGpu = false; -#ifdef HAVE_VAAPI - decodedOnGpu = (dec_frame->format == AV_PIX_FMT_VAAPI); -#endif + // A frame left in GPU memory carries the pool it came from, and + // only such a frame does - which settles whether the GPU really + // decoded it, whatever was asked for. + const bool decodedOnGpu = dec_frame->hw_frames_ctx != nullptr; if (!reportedAcceleration) { - emit accelerationInUse(decodedOnGpu ? QStringLiteral("GPU decoding (VAAPI)") - : QStringLiteral("CPU decoding")); + emit accelerationInUse(decodedOnGpu && hwDecoderName + ? QString("GPU decoding (%1)").arg(QLatin1String(hwDecoderName)) + : QStringLiteral("CPU decoding")); reportedAcceleration = true; } @@ -1864,10 +2059,11 @@ void VideoDevice::Play() // that can draw it, it has to be brought down into system memory // first. AVFrame *display_frame = dec_frame; -#ifdef HAVE_VAAPI AVFrame *sw_frame = nullptr; - if (decodedOnGpu && m_useVaapiRender && !m_hwRenderDisabled.loadRelaxed()) +#ifdef HAVE_VAAPI + if (decodedOnGpu && dec_frame->format == AV_PIX_FMT_VAAPI && + m_useVaapiRender && !m_hwRenderDisabled.loadRelaxed()) { // Zero copy: hand the surface itself over, holding a // reference so it stays alive until the renderer is done. @@ -1885,6 +2081,7 @@ void VideoDevice::Play() av_frame_unref(dec_frame); continue; } +#endif if (decodedOnGpu) { @@ -1904,7 +2101,6 @@ void VideoDevice::Play() display_frame = sw_frame; } -#endif // The picture is converted straight to the size it is going to // be drawn at, rather than to its own size for the window to @@ -1954,9 +2150,7 @@ void VideoDevice::Play() qWarning() << "Could not set up playback scaler"; loggedScaleError = true; } -#ifdef HAVE_VAAPI if (sw_frame) av_frame_free(&sw_frame); -#endif av_frame_unref(dec_frame); continue; } @@ -1972,9 +2166,7 @@ void VideoDevice::Play() if (img.isNull()) { -#ifdef HAVE_VAAPI if (sw_frame) av_frame_free(&sw_frame); -#endif av_frame_unref(dec_frame); continue; } @@ -1987,9 +2179,7 @@ void VideoDevice::Play() emit frameReady(img); -#ifdef HAVE_VAAPI if (sw_frame) av_frame_free(&sw_frame); -#endif av_frame_unref(dec_frame); } } @@ -2057,6 +2247,75 @@ QList<MediaDeviceInfo> VideoDevice::Enumerate(enum MediaDeviceStreamDirection di return result; } +bool VideoDevice::AccelerationBuiltIn(VideoAcceleration acceleration, bool encoding) +{ + // The device type a decoder has to support to be driven through this + // acceleration; none for one that has no decoding side. + AVHWDeviceType deviceType = AV_HWDEVICE_TYPE_NONE; + + switch (acceleration) + { + case VideoAccelerationVaapi: + deviceType = AV_HWDEVICE_TYPE_VAAPI; + break; +#ifdef HAVE_D3D12VA + case VideoAccelerationD3d12va: + deviceType = AV_HWDEVICE_TYPE_D3D12VA; + break; +#endif + case VideoAccelerationD3d11va: + deviceType = AV_HWDEVICE_TYPE_D3D11VA; + break; + case VideoAccelerationDxva2: + deviceType = AV_HWDEVICE_TYPE_DXVA2; + break; + default: + break; + } + + for (unsigned int i = 0; i < sizeof(kVideoEncoders) / sizeof(kVideoEncoders[0]); i++) + { + if (encoding) + { + const char *name = nullptr; + + if (acceleration == VideoAccelerationVaapi) + name = kVideoEncoders[i].vaapiEncoder; + else if (acceleration == VideoAccelerationD3d12va) + name = kVideoEncoders[i].d3d12vaEncoder; + else if (acceleration == VideoAccelerationMediaFoundation) + name = kVideoEncoders[i].mediaFoundationEncoder; + + if (name && avcodec_find_encoder_by_name(name)) + return true; + + continue; + } + + if (deviceType == AV_HWDEVICE_TYPE_NONE) + return false; + + // A decoder lists a hardware configuration for each hwaccel compiled + // in for it, and a hwaccel is only compiled in along with the + // hardware context it needs - so finding one here answers for both. + const AVCodec *dec = avcodec_find_decoder(kVideoEncoders[i].id); + + for (int c = 0; dec; c++) + { + const AVCodecHWConfig *config = avcodec_get_hw_config(dec, c); + + if (!config) + break; + + if (config->device_type == deviceType && + (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) + return true; + } + } + + return false; +} + QString VideoDevice::PlatformDriverName() { #if defined(Q_OS_LINUX) diff --git a/pica-client/videodevice.h b/pica-client/videodevice.h index 5c4585c..3119652 100644 --- a/pica-client/videodevice.h +++ b/pica-client/videodevice.h @@ -18,6 +18,7 @@ #define VIDEODEVICE_H #include "mediadevice.h" +#include "callsettings.h" #include "vaapi.h" #include <QObject> #include <QString> @@ -173,22 +174,31 @@ public: // looking for one to forward, so choosing a codec the camera can produce // itself gets that one rather than merely a compatible one. // - // useVaapi asks for the frames to be encoded on the GPU, through whichever - // accelerator the build has - VAAPI where there is libva, Direct3D 12 on - // Windows. It is a request, not a guarantee: with no such support built - // in, without usable hardware, or if the hardware encoder will not open, - // capture falls back to encoding in software. + // acceleration is a set of VideoAcceleration flags naming the hardware + // encoders that may be tried, in this order when several are set: VAAPI, + // Direct3D 12, Media Foundation. Each is a request, not a guarantee: one not + // built in, without usable hardware, or that will not open is passed over + // for the next, and with none left capture encodes in software. Flags for + // decoding-only paths are ignored. Q_INVOKABLE void configureCapture(QString deviceName, int width, int height, int frameRate, bool preferCompressed, QString codec, int bitrate, - bool useVaapi); - - // useVaapi decodes on the GPU. useVaapiRender additionally keeps the - // decoded frames there and emits them through hwFrameReady() instead of - // frameReady(), for a renderer that can draw a VA surface directly; it - // implies GPU decoding, since there is no surface to draw otherwise. Both - // fall back to software the same way capture does. + int acceleration); + + // acceleration is the decoding counterpart of configureCapture()'s: the + // VideoAcceleration flags for the GPU decoders that may be tried, in this + // order: VAAPI, Direct3D 11, Direct3D 12, DXVA2. The first whose device + // comes up is the one used - whether the GPU can decode the particular + // stream is only found out from the stream itself, and a GPU that cannot + // leaves decoding in software rather than moving on to the next API. + // + // useVaapiRender additionally keeps the decoded frames on the GPU and emits + // them through hwFrameReady() instead of frameReady(), for a renderer that + // can draw a VA surface directly; it implies VAAPI decoding, since there is + // no surface to draw otherwise. The Windows decoders always bring their + // frames back to system memory - nothing in the client can draw a Direct3D + // surface yet. Q_INVOKABLE void configurePlayback(QString codec, int width, int height, - bool useVaapi, bool useVaapiRender); + int acceleration, bool useVaapiRender); // Push one complete encoded frame (already reassembled from 0x77 // fragments by the caller) into the decode queue. Safe to call from any @@ -312,6 +322,14 @@ public: // the same way it may refuse any other request. static QList<VideoCaptureFormat> CaptureFormats(const QString &device); + // Whether the FFmpeg the client runs with was built with what one + // acceleration flag needs in the given direction: a hardware encoder for at + // least one of the call codecs, or the hardware context and a hwaccel for + // at least one of their decoders. Says nothing about whether this machine's + // GPU can use it - only whether there is anything to try at all, which is + // what the settings dialog needs to know before offering it. + static bool AccelerationBuiltIn(VideoAcceleration acceleration, bool encoding); + private: QString m_deviceName; QString m_codec; @@ -320,7 +338,9 @@ private: int m_frameRate; int m_bitrate; bool m_preferCompressed; - bool m_useVaapi; + // VideoAcceleration flags; which of them apply depends on the direction + // this instance was configured for. + int m_acceleration; bool m_useVaapiRender; QAtomicInt m_abort; diff --git a/windows/cross_build_mxe.sh b/windows/cross_build_mxe.sh index fd9a802..288068d 100755 --- a/windows/cross_build_mxe.sh +++ b/windows/cross_build_mxe.sh @@ -227,7 +227,7 @@ test -x ../../configure || die "../../configure not found, run ./autogen.sh in t rm -rf build_pica-pica mkdir -p build_pica-pica cd build_pica-pica -../../../configure --host=$arch --prefix=$PREFIX +../../../configure --host=$arch --prefix=$PREFIX --enable-d3d12va make -j$(nproc) make install make clean diff --git a/windows/mxe_ffmpeg_strip_excess_deps.patch b/windows/mxe_ffmpeg_strip_excess_deps.patch index 5bd901a..9ed8508 100644 --- a/windows/mxe_ffmpeg_strip_excess_deps.patch +++ b/windows/mxe_ffmpeg_strip_excess_deps.patch @@ -30,9 +30,9 @@ index 8dd80d8..4c7daec 100644 +# package provides their headers yet, so configure prints a warning for each of +# them and leaves them out. That is deliberate: adding ffnvcodec (NVIDIA), +# libvpl (Intel QSV) or the AMF headers as a dependency is then the only change -+# needed to get them. The MediaFoundation encoders (h264_mf, hevc_mf) and the -+# dxva2/d3d11va/d3d12va hardware decoders need nothing but mingw-w64 and are -+# available right away. ++# needed to get them. The Media Foundation and Direct3D 12 encoders (h264_mf, ++# hevc_mf, h264_d3d12va, hevc_d3d12va) and the dxva2/d3d11va/d3d12va decoders ++# need nothing but mingw-w64 and are available right away. define $(PKG)_BUILD cd '$(BUILD_DIR)' && '$(SOURCE_DIR)/configure' \ --cross-prefix='$(TARGET)'- \ @@ -72,7 +72,7 @@ index 8dd80d8..4c7daec 100644 + --enable-decoder=h264,hevc,vp9,libvpx_vp9,mjpeg,rawvideo,opus,libopus,adpcm_g722,pcm_alaw,pcm_mulaw,pcm_s16le \ + --enable-encoder=libx264,libx265,libvpx_vp9,mjpeg,libopus,opus,adpcm_g722,pcm_alaw,pcm_mulaw,pcm_s16le \ + --enable-hwaccel=h264_dxva2,h264_d3d11va,h264_d3d11va2,h264_d3d12va,hevc_dxva2,hevc_d3d11va,hevc_d3d11va2,hevc_d3d12va,vp9_dxva2,vp9_d3d11va,vp9_d3d11va2,vp9_d3d12va \ -+ --enable-encoder=h264_mf,hevc_mf \ ++ --enable-encoder=h264_mf,hevc_mf,h264_d3d12va,hevc_d3d12va \ + --enable-hwaccel=h264_nvdec,hevc_nvdec,vp9_nvdec,mjpeg_nvdec \ + --enable-decoder=h264_qsv,hevc_qsv,vp9_qsv,mjpeg_qsv,h264_cuvid,hevc_cuvid,vp9_cuvid,mjpeg_cuvid \ + --enable-encoder=h264_nvenc,hevc_nvenc,h264_qsv,hevc_qsv,h264_amf,hevc_amf \ |
