diff options
| -rw-r--r-- | Makefile.am | 29 | ||||
| -rw-r--r-- | configure.ac | 32 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 18 | ||||
| -rw-r--r-- | pica-client/callwindow.cpp | 26 | ||||
| -rw-r--r-- | pica-client/callwindow.h | 11 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 124 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.h | 21 | ||||
| -rw-r--r-- | pica-client/main.cpp | 7 | ||||
| -rw-r--r-- | pica-client/vaapi.cpp | 101 | ||||
| -rw-r--r-- | pica-client/vaapi.h | 70 | ||||
| -rw-r--r-- | pica-client/vaapivideowidget.cpp | 398 | ||||
| -rw-r--r-- | pica-client/vaapivideowidget.h | 97 | ||||
| -rw-r--r-- | pica-client/videodevice.cpp | 347 | ||||
| -rw-r--r-- | pica-client/videodevice.h | 33 |
14 files changed, 1257 insertions, 57 deletions
diff --git a/Makefile.am b/Makefile.am index 5b5cf91..88e9853 100644 --- a/Makefile.am +++ b/Makefile.am @@ -159,14 +159,20 @@ moc_audiodevice.cpp \ moc_videodevice.cpp \ ui_mainwindow.h +# moc runs its own preprocessor and sees none of the compiler's defines, so +# any define that a header's signals or slots are conditional on has to be +# repeated here - otherwise moc skips those declarations and the signals end +# up declared but never generated, failing at link time. +MOCFLAGS = + moc_%.cpp: pica-client/%.h - $(MOC) $< -o $@ + $(MOC) $(MOCFLAGS) $< -o $@ moc_%.cpp: pica-client/dialogs/%.h - $(MOC) $< -o $@ + $(MOC) $(MOCFLAGS) $< -o $@ moc_%.cpp: pica-client/toneplayer/%.h - $(MOC) $< -o $@ + $(MOC) $(MOCFLAGS) $< -o $@ pica-client/mainwindow.cpp: ui_mainwindow.h @@ -180,6 +186,23 @@ ui_%.h: pica-client/%.ui pica_client_pica_client_CPPFLAGS = -DPICA_INSTALLPREFIX=\"$(prefix)\" $(QT_DEPENDENCIES_CFLAGS) $(FFMPEG_CFLAGS) $(PLATFORM_DEPS_CFLAGS) pica_client_pica_client_LDADD = $(QT_DEPENDENCIES_LIBS) $(FFMPEG_LIBS) $(PLATFORM_DEPS_LIBS) +# VAAPI accelerated encoding, decoding and rendering. Everything below is left +# out of the build entirely when configure did not find libva/EGL or was told +# not to look - the code itself is additionally guarded by #ifdef HAVE_VAAPI. +if PICA_VAAPI +pica_client_pica_client_SOURCES += \ +pica-client/vaapi.cpp \ +pica-client/vaapi.h \ +pica-client/vaapivideowidget.cpp \ +pica-client/vaapivideowidget.h + +nodist_pica_client_pica_client_SOURCES += moc_vaapivideowidget.cpp + +pica_client_pica_client_CPPFLAGS += $(VAAPI_CFLAGS) +pica_client_pica_client_LDADD += $(VAAPI_LIBS) -lGL +MOCFLAGS += -DHAVE_VAAPI +endif + pkgpicaclientdir = ${datarootdir}/pica-client dist_pkgpicaclient_DATA = \ dhparam4096.pem \ diff --git a/configure.ac b/configure.ac index 70a914d..b8bcc02 100644 --- a/configure.ac +++ b/configure.ac @@ -82,8 +82,18 @@ AS_IF([test "x$enable_client" != "xno"], [ PICA_CLIENT_ENABLED=yes ]) +#checking if VAAPI hardware acceleration is wanted +#unspecified means autodetect: use it when the libraries are there, do without +#them when they are not. --enable-vaapi makes their absence a hard error, +#--disable-vaapi skips looking for them at all. +AC_ARG_ENABLE([vaapi], + AS_HELP_STRING([--enable-vaapi], + [Enable VAAPI accelerated video encoding, decoding and rendering (default: autodetect)]) + AS_HELP_STRING([--disable-vaapi], + [Disable VAAPI accelerated video encoding, decoding and rendering])) + #at least one program must be enabled -AS_IF([test "x$PICA_NODE_ENABLED" != "xyes" && test "x$PICA_CLIENT_ENABLED" != "xyes" ], +AS_IF([test "x$PICA_NODE_ENABLED" != "xyes" && test "x$PICA_CLIENT_ENABLED" != "xyes" ], AC_MSG_ERROR(["Either pica-node or pica-client must be enabled"]) ) @@ -131,7 +141,27 @@ AS_IF([test "x$OPENSSL_TOOL" != xyes],[AC_MSG_WARN(["openssl command line tool n PKG_CHECK_MODULES([FFMPEG],["libavcodec libavdevice libavformat libavutil libswresample libswscale"]) AC_SUBST([FFMPEG_CFLAGS]) AC_SUBST([FFMPEG_LIBS]) + +#check for VAAPI libraries, unless explicitly disabled +#libva is the accelerator itself, libva-drm opens it through a DRM render node, +#and EGL is what lets a decoded surface be handed to OpenGL without a copy. +AS_IF([test "x$enable_vaapi" != "xno"], [ + PKG_CHECK_MODULES([VAAPI], [libva libva-drm egl], + [ + PICA_VAAPI_ENABLED=yes + AC_DEFINE([HAVE_VAAPI], [1], [Define to 1 if VAAPI accelerated video is available]) + AC_SUBST([VAAPI_CFLAGS]) + AC_SUBST([VAAPI_LIBS]) + ], + [ + AS_IF([test "x$enable_vaapi" = "xyes"], + [AC_MSG_ERROR(["VAAPI support was requested but libva, libva-drm or EGL was not found"])], + [AC_MSG_WARN(["libva, libva-drm or EGL not found - building without VAAPI acceleration"])]) + ]) ]) +]) + +AM_CONDITIONAL([PICA_VAAPI], [test "x$PICA_VAAPI_ENABLED" = "xyes"]) AC_SUBST([PLATFORM_DEPS_CFLAGS]) AC_SUBST([PLATFORM_DEPS_LIBS]) diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index 54dfa26..59ffad9 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -210,6 +210,7 @@ void AudioVideoCallController::startVideoPipeline() m_videoFrameStarted = false; bool preferCompressed = st.loadValue("video.prefer_compressed", 0).toBool(); + bool vaapiEncoding = st.loadValue("video.vaapi_encoding", 0).toBool(); // The peer is told what we are sending only once the camera is open and // the format is settled - with a compressed camera stream being forwarded @@ -218,7 +219,7 @@ void AudioVideoCallController::startVideoPipeline() QMetaObject::invokeMethod(cam, "configureCapture", Qt::QueuedConnection, Q_ARG(QString, camDev), Q_ARG(int, kVideoWidth), Q_ARG(int, kVideoHeight), - Q_ARG(bool, preferCompressed)); + Q_ARG(bool, preferCompressed), Q_ARG(bool, vaapiEncoding)); QMetaObject::invokeMethod(cam, "Capture", Qt::QueuedConnection); // remotevideo is configured lazily, once the peer's own 0x75 (see @@ -310,6 +311,9 @@ void AudioVideoCallController::start_call(QByteArray peer_id) /* decoded frames arrive from remotevideo's thread, so this is a queued connection; it dies with the window, which is deleted on close */ connect(remotevideo, SIGNAL(frameReady(QImage)), callwindow, SLOT(showRemoteFrame(QImage))); +#ifdef HAVE_VAAPI + connect(remotevideo, SIGNAL(hwFrameReady(AVFramePtr)), callwindow, SLOT(showRemoteHwFrame(AVFramePtr))); +#endif callwindow->show(); } @@ -327,8 +331,11 @@ void AudioVideoCallController::call_from(QByteArray peer_id) connect(callwindow, SIGNAL(accept_call_pressed()), this, SLOT(accept_call())); connect(callwindow, SIGNAL(hang_call_pressed()), this, SLOT(end_call())); connect(callwindow, SIGNAL(callwindow_closed(CallWindow*)), this, SLOT(callwindow_closed(CallWindow*))); - /* see the same connection in start_call() */ + /* see the same connections in start_call() */ connect(remotevideo, SIGNAL(frameReady(QImage)), callwindow, SLOT(showRemoteFrame(QImage))); +#ifdef HAVE_VAAPI + connect(remotevideo, SIGNAL(hwFrameReady(AVFramePtr)), callwindow, SLOT(showRemoteHwFrame(AVFramePtr))); +#endif callwindow->show(); /* play ring tone */ @@ -456,8 +463,13 @@ void AudioVideoCallController::incoming_video_params(QByteArray peer_id, QString if (!is_active || peer_id != m_peer_id) return; + Settings st(config_dbname); + bool vaapiDecoding = st.loadValue("video.vaapi_decoding", 0).toBool(); + bool vaapiRendering = st.loadValue("video.vaapi_rendering", 0).toBool(); + QMetaObject::invokeMethod(remotevideo, "configurePlayback", Qt::QueuedConnection, - Q_ARG(QString, codec), Q_ARG(int, (int)width), Q_ARG(int, (int)height)); + Q_ARG(QString, codec), Q_ARG(int, (int)width), Q_ARG(int, (int)height), + Q_ARG(bool, vaapiDecoding), Q_ARG(bool, vaapiRendering)); QMetaObject::invokeMethod(remotevideo, "Play", Qt::QueuedConnection); } diff --git a/pica-client/callwindow.cpp b/pica-client/callwindow.cpp index 5eb21cc..b2975d5 100644 --- a/pica-client/callwindow.cpp +++ b/pica-client/callwindow.cpp @@ -38,11 +38,19 @@ CallWindow::CallWindow(QByteArray peer_id, bool incoming) lbVideo->setAlignment(Qt::AlignCenter); lbVideo->setMinimumSize(320, 240); lbVideo->hide(); +#ifdef HAVE_VAAPI + videoGpu = new VaapiVideoWidget(this); + videoGpu->setMinimumSize(320, 240); + videoGpu->hide(); +#endif callTimer = new QTimer(this); callElapsedSeconds = 0; lv->addWidget(lbTimer); lv->addWidget(lbVideo); +#ifdef HAVE_VAAPI + lv->addWidget(videoGpu); +#endif lh->addWidget(pbAccept, Qt::AlignLeft); lh->addWidget(pbCall, Qt::AlignLeft); @@ -95,6 +103,24 @@ void CallWindow::showRemoteFrame(QImage frame) Qt::SmoothTransformation)); } +#ifdef HAVE_VAAPI +void CallWindow::showRemoteHwFrame(AVFramePtr frame) +{ + if (!frame) + return; + + // These frames hold a GPU surface rather than pixels, so the label cannot + // show them - the GL widget takes over the video area instead. + if (videoGpu->isHidden()) + { + lbVideo->hide(); + videoGpu->show(); + } + + videoGpu->setFrame(frame); +} +#endif + void CallWindow::update_call_timer() { int hours, minutes, seconds; diff --git a/pica-client/callwindow.h b/pica-client/callwindow.h index 9b9869e..b517cd4 100644 --- a/pica-client/callwindow.h +++ b/pica-client/callwindow.h @@ -23,6 +23,9 @@ #include <QLabel> #include <QTimer> #include <QImage> +#ifdef HAVE_VAAPI +#include "vaapivideowidget.h" +#endif class CallWindow : public QWidget { @@ -44,6 +47,11 @@ private: // Displays the remote side's video. Stays hidden until the first frame // arrives, so an audio-only call keeps the compact window it had before. QLabel *lbVideo; +#ifdef HAVE_VAAPI + // Used instead of lbVideo when frames arrive still living in GPU memory; + // only one of the two is ever visible. + VaapiVideoWidget *videoGpu; +#endif QTimer *callTimer; int callElapsedSeconds; @@ -53,6 +61,9 @@ public slots: void call_started(); void call_ended(); void showRemoteFrame(QImage frame); +#ifdef HAVE_VAAPI + void showRemoteHwFrame(AVFramePtr frame); +#endif private slots: void call(); void accept(); diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index 7a7f868..6aa7a23 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -228,6 +228,15 @@ SettingsDialog::SettingsDialog(QWidget *parent) : cbPreferCompressed = new QCheckBox(tr("Prefer compressed formats if provided by the camera"), this); +#ifdef HAVE_VAAPI + cbVaapiEncoding = new QCheckBox(tr("enable VAAPI-accelerated hardware encoding"), this); + cbVaapiDecoding = new QCheckBox(tr("enable VAAPI-accelerated hardware decoding"), this); + cbVaapiRendering = new QCheckBox(tr("enable VAAPI accelerated hardware rendering"), this); + // Drawing a VA surface means having decoded into one first, so rendering + // brings decoding along with it whether or not it was asked for. + cbVaapiRendering->setToolTip(tr("Requires hardware decoding, which is enabled along with it")); +#endif + btVideoTest = new QPushButton(tr("Test 📷"), this); connect(btVideoTest, SIGNAL(clicked()), this, SLOT(toggleVideoTest())); @@ -236,11 +245,30 @@ SettingsDialog::SettingsDialog(QWidget *parent) : videoPreview->setMinimumSize(320, 240); videoPreview->setFrameShape(QFrame::StyledPanel); + videoTestStatus = new QLabel(this); + videoTestStatus->setAlignment(Qt::AlignCenter); + +#ifdef HAVE_VAAPI + videoPreviewGpu = new VaapiVideoWidget(this); + videoPreviewGpu->setMinimumSize(320, 240); + videoPreviewGpu->hide(); + connect(videoPreviewGpu, SIGNAL(renderingFailed(QString)), this, SLOT(videoTestRenderFailed(QString))); +#endif + videodevLayout->addWidget(videoDev); videodevLayout->addWidget(videoDevRefresh); videodevLayout->addWidget(cbPreferCompressed); +#ifdef HAVE_VAAPI + videodevLayout->addWidget(cbVaapiEncoding); + videodevLayout->addWidget(cbVaapiDecoding); + videodevLayout->addWidget(cbVaapiRendering); +#endif videodevLayout->addWidget(btVideoTest); videodevLayout->addWidget(videoPreview); +#ifdef HAVE_VAAPI + videodevLayout->addWidget(videoPreviewGpu); +#endif + videodevLayout->addWidget(videoTestStatus); videodevLayout->addStretch(1); videodevtab->setLayout(videodevLayout); @@ -255,12 +283,17 @@ SettingsDialog::SettingsDialog(QWidget *parent) : connect(testCam, SIGNAL(captureStarted(QString,int,int)), this, SLOT(videoTestCaptureStarted(QString,int,int))); connect(testCam, SIGNAL(packetReady(QByteArray,bool)), this, SLOT(videoTestFragment(QByteArray,bool))); connect(testCam, SIGNAL(errorOccurred(QString)), this, SLOT(videoTestError(QString))); + connect(testCam, SIGNAL(accelerationInUse(QString)), this, SLOT(videoTestPath(QString))); testCamThread.start(); testDecoder = new VideoDevice(); testDecoder->moveToThread(&testDecoderThread); connect(testDecoder, SIGNAL(frameReady(QImage)), this, SLOT(videoTestFrame(QImage))); connect(testDecoder, SIGNAL(errorOccurred(QString)), this, SLOT(videoTestError(QString))); + connect(testDecoder, SIGNAL(accelerationInUse(QString)), this, SLOT(videoTestPath(QString))); +#ifdef HAVE_VAAPI + connect(testDecoder, SIGNAL(hwFrameReady(AVFramePtr)), this, SLOT(videoTestHwFrame(AVFramePtr))); +#endif testDecoderThread.start(); tabW->addTab(directc2ctab, tr("Direct Connections")); @@ -538,10 +571,18 @@ void SettingsDialog::toggleVideoTest() btVideoTest->setText(tr("Stop ⏹")); videoPreview->setText(tr("Starting...")); + bool vaapiEncoding = false; +#ifdef HAVE_VAAPI + vaapiEncoding = cbVaapiEncoding->isChecked(); +#endif + + videoTestPathReport.clear(); + QMetaObject::invokeMethod(testCam, "configureCapture", Qt::QueuedConnection, Q_ARG(QString, dev), Q_ARG(int, kVideoTestWidth), Q_ARG(int, kVideoTestHeight), - Q_ARG(bool, cbPreferCompressed->isChecked())); + Q_ARG(bool, cbPreferCompressed->isChecked()), + Q_ARG(bool, vaapiEncoding)); QMetaObject::invokeMethod(testCam, "Capture", Qt::QueuedConnection); // The decoder is only started once the camera has settled on a format, @@ -556,14 +597,35 @@ void SettingsDialog::videoTestCaptureStarted(QString codec, int width, int heigh // Says which path the test is exercising: the camera's own compressed // stream forwarded untouched, or frames decoded and re-encoded here. - videoPreview->setText(tr("Capturing %1 %2x%3...").arg(codec).arg(width).arg(height)); + videoTestFormat = tr("%1 %2x%3").arg(codec).arg(width).arg(height); + videoPreview->setText(tr("Capturing %1...").arg(videoTestFormat)); + + bool vaapiDecoding = false; + bool vaapiRendering = false; +#ifdef HAVE_VAAPI + vaapiDecoding = cbVaapiDecoding->isChecked(); + vaapiRendering = cbVaapiRendering->isChecked(); +#endif QMetaObject::invokeMethod(testDecoder, "configurePlayback", Qt::QueuedConnection, Q_ARG(QString, codec), - Q_ARG(int, width), Q_ARG(int, height)); + Q_ARG(int, width), Q_ARG(int, height), + Q_ARG(bool, vaapiDecoding), Q_ARG(bool, vaapiRendering)); QMetaObject::invokeMethod(testDecoder, "Play", Qt::QueuedConnection); } +void SettingsDialog::videoTestPath(QString description) +{ + if (!videoTestRunning) + return; + + // Capture and playback each report once; both lines are kept so the whole + // path is visible, since either end can quietly fall back to software. + if (!videoTestPathReport.isEmpty()) + videoTestPathReport += QLatin1String(", "); + videoTestPathReport += description; +} + void SettingsDialog::stopVideoTest() { // Direct calls: both devices are blocked inside their capture/decode @@ -574,8 +636,16 @@ void SettingsDialog::stopVideoTest() testAssembler.reset(); videoTestRunning = false; + videoTestFormat.clear(); + videoTestPathReport.clear(); btVideoTest->setText(tr("Test 📷")); videoPreview->clear(); + videoTestStatus->clear(); +#ifdef HAVE_VAAPI + videoPreviewGpu->clearFrame(); + videoPreviewGpu->hide(); + videoPreview->show(); +#endif } void SettingsDialog::videoTestFragment(QByteArray data, bool is_last_fragment) @@ -602,11 +672,48 @@ void SettingsDialog::videoTestFrame(QImage frame) if (!videoTestRunning || frame.isNull()) return; + // The preview itself is showing pixels now, so what format came off the + // camera and which of the GPU or CPU did the work goes underneath it. + videoTestStatus->setText(videoTestPathReport.isEmpty() + ? videoTestFormat + : tr("%1 - %2").arg(videoTestFormat, videoTestPathReport)); + videoPreview->setPixmap(QPixmap::fromImage(frame).scaled(videoPreview->size(), Qt::KeepAspectRatio, Qt::SmoothTransformation)); } +#ifdef HAVE_VAAPI +void SettingsDialog::videoTestHwFrame(AVFramePtr frame) +{ + if (!videoTestRunning) + return; + + // Frames arriving here never went through system memory, so the label has + // nothing to show - the GL widget takes over the preview area. + if (videoPreviewGpu->isHidden()) + { + videoPreview->hide(); + videoPreviewGpu->show(); + } + + videoTestStatus->setText(videoTestPathReport.isEmpty() + ? videoTestFormat + : tr("%1 - %2, GPU rendering").arg(videoTestFormat, videoTestPathReport)); + + videoPreviewGpu->setFrame(frame); +} + +void SettingsDialog::videoTestRenderFailed(QString message) +{ + // Drawing straight from GPU memory did not work out. The decoder is still + // running, so say so and leave the test going rather than killing it. + videoPreviewGpu->hide(); + videoPreview->show(); + videoPreview->setText(message); +} +#endif + void SettingsDialog::videoTestError(QString message) { if (!videoTestRunning) @@ -776,6 +883,12 @@ void SettingsDialog::loadSettings() cbPreferCompressed->setChecked(st.loadValue("video.prefer_compressed", 0).toBool()); +#ifdef HAVE_VAAPI + cbVaapiEncoding->setChecked(st.loadValue("video.vaapi_encoding", 0).toBool()); + cbVaapiDecoding->setChecked(st.loadValue("video.vaapi_decoding", 0).toBool()); + cbVaapiRendering->setChecked(st.loadValue("video.vaapi_rendering", 0).toBool()); +#endif + QString audioCapDevVal = st.loadValue("audio.capture_device", "default").toString(); int audioCapDevItem = audioCaptureDev->findData(audioCapDevVal); if (audioCapDevItem >= 0) @@ -830,6 +943,11 @@ void SettingsDialog::storeSettings() st.storeValue("video.capture_device", videoDev->itemData(videoDev->currentIndex()).toString()); st.storeValue("video.prefer_compressed", cbPreferCompressed->isChecked() ? "1" : "0"); +#ifdef HAVE_VAAPI + st.storeValue("video.vaapi_encoding", cbVaapiEncoding->isChecked() ? "1" : "0"); + st.storeValue("video.vaapi_decoding", cbVaapiDecoding->isChecked() ? "1" : "0"); + st.storeValue("video.vaapi_rendering", cbVaapiRendering->isChecked() ? "1" : "0"); +#endif st.storeValue("audio.capture_device", audioCaptureDev->itemData(audioCaptureDev->currentIndex()).toString()); st.storeValue("audio.playback_device", audioPlaybackDev->itemData(audioPlaybackDev->currentIndex()).toString()); st.storeValue("audio.ring_device", audioRingDev->itemData(audioRingDev->currentIndex()).toString()); diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h index a6e6832..e3ebea7 100644 --- a/pica-client/dialogs/settingsdialog.h +++ b/pica-client/dialogs/settingsdialog.h @@ -33,6 +33,9 @@ #include "../audiodevice.h" #include "../videodevice.h" #include "../toneplayer/toneplayer.h" +#ifdef HAVE_VAAPI +#include "../vaapivideowidget.h" +#endif class SettingsDialog : public QDialog { @@ -118,7 +121,20 @@ private: // the network, so the whole chain can be checked from this dialog. QPushButton *btVideoTest; QLabel *videoPreview; + // Shown under the preview: the format the camera settled on, and whether + // the GPU or the CPU ended up doing the encoding and decoding. + QLabel *videoTestStatus; + QString videoTestFormat; + QString videoTestPathReport; QCheckBox *cbPreferCompressed; +#ifdef HAVE_VAAPI + QCheckBox *cbVaapiEncoding; + QCheckBox *cbVaapiDecoding; + QCheckBox *cbVaapiRendering; + // Shown in place of videoPreview while frames are being drawn straight + // from GPU memory; only one of the two is ever visible. + VaapiVideoWidget *videoPreviewGpu; +#endif VideoDevice *testCam; VideoDevice *testDecoder; QThread testCamThread; @@ -159,8 +175,13 @@ private slots: void toggleVideoTest(); void videoTestCaptureStarted(QString codec, int width, int height); + void videoTestPath(QString description); void videoTestFragment(QByteArray data, bool is_last_fragment); void videoTestFrame(QImage frame); +#ifdef HAVE_VAAPI + void videoTestHwFrame(AVFramePtr frame); + void videoTestRenderFailed(QString message); +#endif void videoTestError(QString message); }; diff --git a/pica-client/main.cpp b/pica-client/main.cpp index 0fae44c..cc3d90b 100644 --- a/pica-client/main.cpp +++ b/pica-client/main.cpp @@ -28,6 +28,7 @@ #include "audiovideocallcontroller.h" #include "picaactioncenter.h" #include "picasystray.h" +#include "vaapi.h" #include <QDir> #include <QMessageBox> #include <QString> @@ -500,6 +501,12 @@ int main(int argc, char *argv[]) { QApplication a(argc, argv); +#ifdef HAVE_VAAPI + // Decoded hardware frames travel from the decoding thread to the widget + // drawing them through queued signals, which needs their type registered. + VaapiContext::registerMetaTypes(); +#endif + #ifdef WIN32 config_resourceDir = QLatin1String("share\\"); #elif defined(__APPLE__) diff --git a/pica-client/vaapi.cpp b/pica-client/vaapi.cpp new file mode 100644 index 0000000..74f036d --- /dev/null +++ b/pica-client/vaapi.cpp @@ -0,0 +1,101 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "vaapi.h" + +#ifdef HAVE_VAAPI + +#include <QMutex> +#include <QMutexLocker> +#include <QDebug> + +extern "C" { +#include <libavutil/hwcontext.h> +#include <libavutil/hwcontext_vaapi.h> +#include <libavutil/error.h> +} + +namespace +{ + QMutex g_mutex; + // Created on first use and kept for the lifetime of the process: opening a + // VAAPI device is not cheap, and every user of it has to share the one + // display anyway. + AVBufferRef *g_device = nullptr; + bool g_tried = false; + + // Must be called with g_mutex held. + void ensureDevice() + { + if (g_tried) + return; + + g_tried = true; + + // A null device name lets FFmpeg find a usable DRM render node + // (/dev/dri/renderD128 and friends) by itself. + int ret = av_hwdevice_ctx_create(&g_device, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0); + if (ret < 0) + { + char buf[256] = {0}; + av_strerror(ret, buf, sizeof(buf)); + qWarning() << "VAAPI is not usable, falling back to software video:" << buf; + g_device = nullptr; + } + } +} + +AVFramePtr wrapAVFrame(AVFrame *frame) +{ + return AVFramePtr(frame, [](AVFrame *f) { av_frame_free(&f); }); +} + +AVBufferRef *VaapiContext::deviceRef() +{ + QMutexLocker locker(&g_mutex); + ensureDevice(); + + return g_device ? av_buffer_ref(g_device) : nullptr; +} + +VADisplay VaapiContext::display() +{ + QMutexLocker locker(&g_mutex); + ensureDevice(); + + if (!g_device) + return nullptr; + + AVHWDeviceContext *devctx = (AVHWDeviceContext *)g_device->data; + AVVAAPIDeviceContext *vactx = (AVVAAPIDeviceContext *)devctx->hwctx; + + return vactx->display; +} + +bool VaapiContext::isAvailable() +{ + QMutexLocker locker(&g_mutex); + ensureDevice(); + + return g_device != nullptr; +} + +void VaapiContext::registerMetaTypes() +{ + qRegisterMetaType<AVFramePtr>("AVFramePtr"); +} + +#endif // HAVE_VAAPI diff --git a/pica-client/vaapi.h b/pica-client/vaapi.h new file mode 100644 index 0000000..c422a88 --- /dev/null +++ b/pica-client/vaapi.h @@ -0,0 +1,70 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#ifndef VAAPI_H +#define VAAPI_H + +#ifdef HAVE_VAAPI + +#include <QSharedPointer> +#include <QMetaType> +#include <va/va.h> + +extern "C" { +#include <libavutil/buffer.h> +#include <libavutil/frame.h> +} + +// A decoded hardware frame holds a VA surface, not pixels, and has to travel +// from the decoding thread to the widget that draws it. Reference counting it +// this way keeps the surface alive exactly as long as someone still needs it, +// and lets it cross threads through an ordinary queued signal. +typedef QSharedPointer<AVFrame> AVFramePtr; + +// Wraps an AVFrame that the caller is handing over, freeing it once the last +// reference goes away. +AVFramePtr wrapAVFrame(AVFrame *frame); + +// The one VAAPI device the whole client shares. Encoder, decoder and renderer +// must all work against the same VADisplay - surfaces created on one display +// cannot be used on another - so the device is created once, on first use, and +// handed out from here. +class VaapiContext +{ +public: + // A new reference to the shared device context, for an encoder's or + // decoder's hw_device_ctx. The caller owns the returned reference and + // unrefs it as usual. Null if VAAPI is unavailable. + static AVBufferRef *deviceRef(); + + // The underlying VADisplay, needed to export a decoded surface for + // display. Null if VAAPI is unavailable. + static VADisplay display(); + + // Whether a VAAPI device could be opened at all. Callers use this to fall + // back to software rather than each probing for themselves. + static bool isAvailable(); + + // Registers the metatype for AVFramePtr, so hardware frames can be passed + // through queued signal connections. Call once at startup. + static void registerMetaTypes(); +}; + +Q_DECLARE_METATYPE(AVFramePtr) + +#endif // HAVE_VAAPI + +#endif // VAAPI_H diff --git a/pica-client/vaapivideowidget.cpp b/pica-client/vaapivideowidget.cpp new file mode 100644 index 0000000..fa50d10 --- /dev/null +++ b/pica-client/vaapivideowidget.cpp @@ -0,0 +1,398 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "vaapivideowidget.h" + +#ifdef HAVE_VAAPI + +#include <QDebug> +#include <unistd.h> + +#include <EGL/egl.h> +#include <EGL/eglext.h> +#include <GL/gl.h> +#include <drm/drm_fourcc.h> + +extern "C" { +#include <libavutil/pixfmt.h> +} + +#include <va/va_drmcommon.h> + +namespace +{ + // Extension entry points, resolved once at first use: they are not + // linkable symbols, only reachable through eglGetProcAddress. + PFNEGLCREATEIMAGEKHRPROC g_eglCreateImageKHR = nullptr; + PFNEGLDESTROYIMAGEKHRPROC g_eglDestroyImageKHR = nullptr; + PFNGLEGLIMAGETARGETTEXTURE2DOESPROC g_glEGLImageTargetTexture2DOES = nullptr; + bool g_resolved = false; + + bool resolveEglExtensions() + { + if (g_resolved) + return g_eglCreateImageKHR && g_eglDestroyImageKHR && g_glEGLImageTargetTexture2DOES; + + g_resolved = true; + g_eglCreateImageKHR = + (PFNEGLCREATEIMAGEKHRPROC)eglGetProcAddress("eglCreateImageKHR"); + g_eglDestroyImageKHR = + (PFNEGLDESTROYIMAGEKHRPROC)eglGetProcAddress("eglDestroyImageKHR"); + g_glEGLImageTargetTexture2DOES = + (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress("glEGLImageTargetTexture2DOES"); + + return g_eglCreateImageKHR && g_eglDestroyImageKHR && g_glEGLImageTargetTexture2DOES; + } + + const char *kVertexShader = + "attribute highp vec2 vertex;\n" + "attribute highp vec2 texcoord;\n" + "varying highp vec2 uv;\n" + "void main() {\n" + " uv = texcoord;\n" + " gl_Position = vec4(vertex, 0.0, 1.0);\n" + "}\n"; + + // NV12 arrives as a full resolution luma plane and a half resolution + // plane of interleaved Cb/Cr. The BT.601 limited range coefficients below + // match what the decoders in this pipeline produce for camera video. + const char *kFragmentShader = + "varying highp vec2 uv;\n" + "uniform sampler2D planeY;\n" + "uniform sampler2D planeUV;\n" + "void main() {\n" + " highp float y = texture2D(planeY, uv).r;\n" + " highp vec2 c = texture2D(planeUV, uv).rg;\n" + " y = 1.1643 * (y - 0.0625);\n" + " highp float u = c.r - 0.5;\n" + " highp float v = c.g - 0.5;\n" + " highp float r = y + 1.5958 * v;\n" + " highp float g = y - 0.39173 * u - 0.81290 * v;\n" + " highp float b = y + 2.017 * u;\n" + " gl_FragColor = vec4(r, g, b, 1.0);\n" + "}\n"; +} + +VaapiVideoWidget::VaapiVideoWidget(QWidget *parent) + : QOpenGLWidget(parent), m_program(nullptr), m_numFds(0), + m_frameWidth(0), m_frameHeight(0), m_working(true), m_reportedFailure(false), + m_describedSurface(false) +{ + m_textures[0] = m_textures[1] = 0; + m_images[0] = m_images[1] = EGL_NO_IMAGE_KHR; + for (int i = 0; i < 4; i++) + m_dmabufFds[i] = -1; +} + +VaapiVideoWidget::~VaapiVideoWidget() +{ + // The GL objects belong to this widget's context, so it has to be current + // while they are destroyed. + makeCurrent(); + releaseImports(); + if (m_textures[0]) + glDeleteTextures(2, m_textures); + delete m_program; + doneCurrent(); +} + +void VaapiVideoWidget::setFrame(AVFramePtr frame) +{ + if (!m_working) + return; + + m_frame = frame; + update(); +} + +void VaapiVideoWidget::clearFrame() +{ + m_frame.clear(); + update(); +} + +void VaapiVideoWidget::initializeGL() +{ + initializeOpenGLFunctions(); + + glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + glDisable(GL_DEPTH_TEST); + + if (!resolveEglExtensions()) + { + m_working = false; + if (!m_reportedFailure) + { + m_reportedFailure = true; + QString msg = "EGL does not offer the extensions needed to draw GPU frames"; + qWarning() << msg; + emit renderingFailed(msg); + } + return; + } + + m_program = new QOpenGLShaderProgram(this); + if (!m_program->addShaderFromSourceCode(QOpenGLShader::Vertex, kVertexShader) || + !m_program->addShaderFromSourceCode(QOpenGLShader::Fragment, kFragmentShader) || + !m_program->link()) + { + m_working = false; + if (!m_reportedFailure) + { + m_reportedFailure = true; + QString msg = QString("Could not build the video shader: %1").arg(m_program->log()); + qWarning() << msg; + emit renderingFailed(msg); + } + return; + } + + glGenTextures(2, m_textures); + for (int i = 0; i < 2; i++) + { + glBindTexture(GL_TEXTURE_2D, m_textures[i]); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + glBindTexture(GL_TEXTURE_2D, 0); +} + +void VaapiVideoWidget::releaseImports() +{ + for (int i = 0; i < 2; i++) + { + if (m_images[i] != EGL_NO_IMAGE_KHR) + { + if (g_eglDestroyImageKHR) + g_eglDestroyImageKHR(eglGetCurrentDisplay(), (EGLImageKHR)m_images[i]); + m_images[i] = EGL_NO_IMAGE_KHR; + } + } + + for (int i = 0; i < m_numFds; i++) + { + if (m_dmabufFds[i] >= 0) + { + ::close(m_dmabufFds[i]); + m_dmabufFds[i] = -1; + } + } + m_numFds = 0; +} + +bool VaapiVideoWidget::importFrame() +{ + VADisplay va_display = VaapiContext::display(); + if (!va_display || !m_frame) + return false; + + // The surface id lives in data[3] for AV_PIX_FMT_VAAPI frames. + VASurfaceID surface = (VASurfaceID)(uintptr_t)m_frame->data[3]; + + // Everything still queued on the surface has to have finished before its + // memory can be handed to another API. + if (vaSyncSurface(va_display, surface) != VA_STATUS_SUCCESS) + return false; + + VADRMPRIMESurfaceDescriptor desc; + memset(&desc, 0, sizeof desc); + + VAStatus st = vaExportSurfaceHandle(va_display, surface, + VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, + VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS, + &desc); + if (st != VA_STATUS_SUCCESS) + return false; + + // Held so they can be closed again after drawing; the export hands over + // ownership of every one of them. + m_numFds = desc.num_objects > 4 ? 4 : (int)desc.num_objects; + for (int i = 0; i < m_numFds; i++) + m_dmabufFds[i] = desc.objects[i].fd; + + m_frameWidth = desc.width; + m_frameHeight = desc.height; + + // Logged once: how the driver actually laid the surface out is the first + // thing worth knowing when the picture comes out wrong. + if (!m_describedSurface) + { + m_describedSurface = true; + + QString layers; + for (uint32_t i = 0; i < desc.num_layers; i++) + { + uint32_t object = desc.layers[i].object_index[0]; + layers += QString(" layer%1[fourcc=0x%2 offset=%3 pitch=%4 modifier=0x%5]") + .arg(i) + .arg(desc.layers[i].drm_format, 0, 16) + .arg(desc.layers[i].offset[0]) + .arg(desc.layers[i].pitch[0]) + .arg(desc.objects[object].drm_format_modifier, 0, 16); + } + + qDebug() << QString("VAAPI surface for display: %1x%2 fourcc=0x%3 objects=%4 layers=%5%6") + .arg(desc.width).arg(desc.height) + .arg(desc.fourcc, 0, 16) + .arg(desc.num_objects).arg(desc.num_layers) + .arg(layers); + } + + // Two layers for NV12: the luma plane, then the interleaved chroma plane + // at half size in both directions. + if (desc.num_layers < 2) + { + releaseImports(); + return false; + } + + EGLDisplay egl_display = eglGetCurrentDisplay(); + + for (int i = 0; i < 2; i++) + { + uint32_t object = desc.layers[i].object_index[0]; + int planeWidth = (i == 0) ? desc.width : desc.width / 2; + int planeHeight = (i == 0) ? desc.height : desc.height / 2; + uint64_t modifier = desc.objects[object].drm_format_modifier; + + EGLint attribs[16]; + int n = 0; + + attribs[n++] = EGL_WIDTH; attribs[n++] = planeWidth; + attribs[n++] = EGL_HEIGHT; attribs[n++] = planeHeight; + attribs[n++] = EGL_LINUX_DRM_FOURCC_EXT; attribs[n++] = (EGLint)desc.layers[i].drm_format; + attribs[n++] = EGL_DMA_BUF_PLANE0_FD_EXT; attribs[n++] = desc.objects[object].fd; + attribs[n++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT; attribs[n++] = (EGLint)desc.layers[i].offset[0]; + attribs[n++] = EGL_DMA_BUF_PLANE0_PITCH_EXT; attribs[n++] = (EGLint)desc.layers[i].pitch[0]; + + // Without this the import is assumed to be a plain linear buffer. GPUs + // hand out tiled surfaces instead, and reading one as linear is what + // produces a picture in bands with the colour gone - so the modifier + // the driver reported has to be passed along whenever it has one. + if (modifier != DRM_FORMAT_MOD_INVALID) + { + attribs[n++] = EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT; + attribs[n++] = (EGLint)(modifier & 0xFFFFFFFF); + attribs[n++] = EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT; + attribs[n++] = (EGLint)(modifier >> 32); + } + + attribs[n++] = EGL_NONE; + + m_images[i] = g_eglCreateImageKHR(egl_display, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, nullptr, attribs); + if (m_images[i] == EGL_NO_IMAGE_KHR) + { + releaseImports(); + return false; + } + + glBindTexture(GL_TEXTURE_2D, m_textures[i]); + g_glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, (GLeglImageOES)m_images[i]); + } + + glBindTexture(GL_TEXTURE_2D, 0); + + return true; +} + +void VaapiVideoWidget::paintGL() +{ + glClear(GL_COLOR_BUFFER_BIT); + + if (!m_working || !m_program || !m_frame) + return; + + if (!importFrame()) + { + m_working = false; + if (!m_reportedFailure) + { + m_reportedFailure = true; + QString msg = "Could not hand the decoded GPU frame to OpenGL for drawing"; + qWarning() << msg; + emit renderingFailed(msg); + } + return; + } + + // Letterbox rather than stretch, matching what the software path does + // with Qt::KeepAspectRatio. + float widgetAspect = float(width()) / float(height() ? height() : 1); + float frameAspect = float(m_frameWidth) / float(m_frameHeight ? m_frameHeight : 1); + float sx = 1.0f, sy = 1.0f; + + if (frameAspect > widgetAspect) + sy = widgetAspect / frameAspect; + else + sx = frameAspect / widgetAspect; + + const GLfloat vertices[] = + { + -sx, -sy, + sx, -sy, + -sx, sy, + sx, sy + }; + // Flipped vertically: video rows run top to bottom, OpenGL's origin is at + // the bottom left. + const GLfloat texcoords[] = + { + 0.0f, 1.0f, + 1.0f, 1.0f, + 0.0f, 0.0f, + 1.0f, 0.0f + }; + + m_program->bind(); + + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, m_textures[0]); + m_program->setUniformValue("planeY", 0); + + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, m_textures[1]); + m_program->setUniformValue("planeUV", 1); + + int vertexLocation = m_program->attributeLocation("vertex"); + int texcoordLocation = m_program->attributeLocation("texcoord"); + + m_program->enableAttributeArray(vertexLocation); + m_program->enableAttributeArray(texcoordLocation); + m_program->setAttributeArray(vertexLocation, vertices, 2); + m_program->setAttributeArray(texcoordLocation, texcoords, 2); + + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + + m_program->disableAttributeArray(vertexLocation); + m_program->disableAttributeArray(texcoordLocation); + m_program->release(); + + glActiveTexture(GL_TEXTURE0); + + // Done with this frame's handles - holding on to them would leak a set of + // file descriptors for every frame drawn. + releaseImports(); +} + +void VaapiVideoWidget::resizeGL(int w, int h) +{ + glViewport(0, 0, w, h); +} + +#endif // HAVE_VAAPI diff --git a/pica-client/vaapivideowidget.h b/pica-client/vaapivideowidget.h new file mode 100644 index 0000000..0aebef5 --- /dev/null +++ b/pica-client/vaapivideowidget.h @@ -0,0 +1,97 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#ifndef VAAPIVIDEOWIDGET_H +#define VAAPIVIDEOWIDGET_H + +#ifdef HAVE_VAAPI + +#include "vaapi.h" + +#include <QOpenGLWidget> +#include <QOpenGLFunctions> +#include <QOpenGLShaderProgram> +#include <QString> + +// Draws frames that are still sitting in GPU memory as VA surfaces, without +// bringing them back through system memory on the way. +// +// The surface is exported as DMA-BUF file descriptors, imported into EGL as +// images, and bound to OpenGL textures - one for the luma plane and one for +// the interleaved chroma plane of an NV12 frame - which a fragment shader +// then converts to RGB while drawing. Nothing copies the picture; the GPU +// keeps it where the decoder left it. +// +// The alternative, vaPutSurface() onto an X11 Drawable, was not an option +// here: the client runs on Wayland, where there is no such drawable, and +// Qt5X11Extras is not installed either. EGL works on both Wayland and X11. +class VaapiVideoWidget : public QOpenGLWidget, protected QOpenGLFunctions +{ + Q_OBJECT +public: + explicit VaapiVideoWidget(QWidget *parent = nullptr); + ~VaapiVideoWidget() override; + + // Whether the last frame could actually be drawn. False means the EGL + // import failed and nothing is being shown, so a caller can fall back to + // the software path rather than leave a blank widget. + bool isWorking() const { return m_working; } + +public slots: + // Takes the frame to draw next, replacing any previous one that has not + // been drawn yet - for live video the newest frame is the one that + // matters, which is the same policy the decode queue follows. + void setFrame(AVFramePtr frame); + + // Drops the current frame and clears the widget. + void clearFrame(); + +signals: + // Raised once if the frame cannot be imported for drawing, carrying the + // reason. The widget stops trying after that. + void renderingFailed(QString message); + +protected: + void initializeGL() override; + void paintGL() override; + void resizeGL(int w, int h) override; + +private: + // Builds the two textures for the current frame out of the surface's + // DMA-BUF handles. Returns false and leaves nothing bound on failure. + bool importFrame(); + + // Releases the EGL images and the file descriptors the export handed + // over. Not closing those descriptors leaks them once per frame, which + // exhausts the process's descriptors within minutes. + void releaseImports(); + + AVFramePtr m_frame; + QOpenGLShaderProgram *m_program; + unsigned int m_textures[2]; + void *m_images[2]; + int m_dmabufFds[4]; + int m_numFds; + int m_frameWidth; + int m_frameHeight; + bool m_working; + bool m_reportedFailure; + bool m_describedSurface; +}; + +#endif // HAVE_VAAPI + +#endif // VAAPIVIDEOWIDGET_H diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp index adf9c6e..e9f71e0 100644 --- a/pica-client/videodevice.cpp +++ b/pica-client/videodevice.cpp @@ -48,6 +48,9 @@ extern "C" { #include <libavutil/mem.h> #include <libavutil/error.h> #include <libswscale/swscale.h> +#ifdef HAVE_VAAPI +#include <libavutil/hwcontext.h> +#endif } // Encoding defaults. Not exposed as settings yet - see the deferred list in @@ -91,6 +94,26 @@ static const struct }; #endif +#ifdef HAVE_VAAPI +// Picks the GPU pixel format out of what the decoder offers. Returning the +// first entry instead - which is what happens when VAAPI is not among them - +// leaves the decoder working in software, which is the wanted fallback. +static AVPixelFormat vaapi_get_format(AVCodecContext *ctx, const AVPixelFormat *formats) +{ + Q_UNUSED(ctx); + + for (const AVPixelFormat *p = formats; *p != AV_PIX_FMT_NONE; p++) + { + if (*p == AV_PIX_FMT_VAAPI) + return *p; + } + + qWarning() << "The GPU cannot decode this stream, decoding in software"; + + return formats[0]; +} +#endif + QStringList VideoDevice::CompressedFormats(const QString &device) { QStringList result; @@ -162,7 +185,8 @@ QByteArray VideoFrameAssembler::addFragment(quint16 seq_num, quint32 timestamp, } VideoDevice::VideoDevice(QObject *parent) - : QObject(parent), m_width(640), m_height(480), m_preferCompressed(false), m_abort(0) + : QObject(parent), m_width(640), m_height(480), m_preferCompressed(false), + m_useVaapi(false), m_useVaapiRender(false), m_abort(0) { } @@ -171,19 +195,26 @@ VideoDevice::~VideoDevice() Close(); } -void VideoDevice::configureCapture(QString deviceName, int width, int height, bool preferCompressed) +void VideoDevice::configureCapture(QString deviceName, int width, int height, + bool preferCompressed, bool useVaapi) { m_deviceName = deviceName; m_width = width; m_height = height; m_preferCompressed = preferCompressed; + m_useVaapi = useVaapi; } -void VideoDevice::configurePlayback(QString codec, int width, int height) +void VideoDevice::configurePlayback(QString codec, int width, int height, + bool useVaapi, bool useVaapiRender) { m_codec = codec; m_width = width; m_height = height; + // Drawing a VA surface requires having decoded into one in the first + // place, so asking for GPU rendering turns on GPU decoding regardless. + m_useVaapi = useVaapi || useVaapiRender; + m_useVaapiRender = useVaapiRender; } void VideoDevice::enqueueFrame(QByteArray encodedFrame) @@ -323,6 +354,10 @@ void VideoDevice::runPassthroughLoop(AVFormatContext *ifmt_ctx, AVStream *in_st, emit captureStarted(codec, width, height); + // Nothing encodes here at all - the camera's own packets go out as they + // arrive - so any hardware encoding setting simply has nothing to act on. + emit accelerationInUse(QStringLiteral("no encoding, camera stream forwarded unchanged")); + AVPacket *pkt = av_packet_alloc(); bool loggedReadError = false; @@ -368,45 +403,129 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) return; } - const AVCodec *enc = avcodec_find_encoder_by_name("libx264"); - if (!enc) - enc = avcodec_find_encoder(AV_CODEC_ID_H264); - AVCodecContext *enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr; - if (!enc || !enc_ctx) + // The GPU encoder is tried first when asked for, and anything that goes + // wrong on the way - no such encoder built into FFmpeg, no usable VAAPI + // device, no surface pool, the encoder refusing to open - just leaves + // hardware false and takes the software path below. + bool hardware = false; + const AVCodec *enc = nullptr; + AVCodecContext *enc_ctx = nullptr; + AVBufferRef *hw_frames_ref = nullptr; + +#ifdef HAVE_VAAPI + if (m_useVaapi && VaapiContext::isAvailable()) { - QString msg = "H.264 encoder not available"; - qWarning() << msg; - emit errorOccurred(msg); - avcodec_free_context(&dec_ctx); - return; + enc = avcodec_find_encoder_by_name("h264_vaapi"); + AVBufferRef *hw_device_ref = enc ? VaapiContext::deviceRef() : nullptr; + + if (hw_device_ref) + { + enc_ctx = avcodec_alloc_context3(enc); + hw_frames_ref = enc_ctx ? av_hwframe_ctx_alloc(hw_device_ref) : nullptr; + + if (hw_frames_ref) + { + // The pool the encoder draws its input surfaces from. NV12 is + // what VAAPI encoders take; the frames themselves live in GPU + // memory, which is what AV_PIX_FMT_VAAPI denotes. + AVHWFramesContext *frames_ctx = (AVHWFramesContext *)hw_frames_ref->data; + frames_ctx->format = AV_PIX_FMT_VAAPI; + frames_ctx->sw_format = AV_PIX_FMT_NV12; + frames_ctx->width = m_width; + frames_ctx->height = m_height; + frames_ctx->initial_pool_size = 20; + + if (av_hwframe_ctx_init(hw_frames_ref) >= 0) + { + enc_ctx->width = m_width; + enc_ctx->height = m_height; + enc_ctx->pix_fmt = AV_PIX_FMT_VAAPI; + enc_ctx->time_base = AVRational{1, kFrameRate}; + enc_ctx->framerate = AVRational{kFrameRate, 1}; + enc_ctx->bit_rate = kBitrate; + enc_ctx->gop_size = kGopSize; + enc_ctx->max_b_frames = 0; + enc_ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref); + + // No preset/tune here: those are x264's own options and + // mean nothing to a VAAPI encoder. + if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) >= 0) + hardware = true; + else + qWarning() << QString("VAAPI H.264 encoder would not open (%1), encoding in software") + .arg(ff_errstr(ret)); + } + else + { + qWarning() << "Could not create a VAAPI surface pool, encoding in software"; + } + } + + av_buffer_unref(&hw_device_ref); + } + else if (enc) + { + qWarning() << "No usable VAAPI device, encoding in software"; + } + else + { + qWarning() << "FFmpeg has no h264_vaapi encoder, encoding in software"; + } + + if (!hardware) + { + if (enc_ctx) avcodec_free_context(&enc_ctx); + if (hw_frames_ref) av_buffer_unref(&hw_frames_ref); + enc = nullptr; + } } +#endif - enc_ctx->width = m_width; - enc_ctx->height = m_height; - enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P; - enc_ctx->time_base = AVRational{1, kFrameRate}; - enc_ctx->framerate = AVRational{kFrameRate, 1}; - enc_ctx->bit_rate = kBitrate; - enc_ctx->gop_size = kGopSize; - // B-frames reorder output, which would cost latency in a live call. - enc_ctx->max_b_frames = 0; - - // Note the absence of AV_CODEC_FLAG_GLOBAL_HEADER: we want the parameter - // sets repeated in-band with every keyframe, since the receiving side has - // no out-of-band way to learn them and may start decoding mid-stream. - av_opt_set(enc_ctx->priv_data, "preset", "veryfast", 0); - av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0); - - if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) < 0) + if (!hardware) { - QString msg = QString("Could not open H.264 encoder: %1").arg(ff_errstr(ret)); - qWarning() << msg; - emit errorOccurred(msg); - avcodec_free_context(&enc_ctx); - avcodec_free_context(&dec_ctx); - return; + enc = avcodec_find_encoder_by_name("libx264"); + if (!enc) + enc = avcodec_find_encoder(AV_CODEC_ID_H264); + enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr; + if (!enc || !enc_ctx) + { + QString msg = "H.264 encoder not available"; + qWarning() << msg; + emit errorOccurred(msg); + avcodec_free_context(&dec_ctx); + return; + } + + enc_ctx->width = m_width; + enc_ctx->height = m_height; + enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P; + enc_ctx->time_base = AVRational{1, kFrameRate}; + enc_ctx->framerate = AVRational{kFrameRate, 1}; + enc_ctx->bit_rate = kBitrate; + enc_ctx->gop_size = kGopSize; + // B-frames reorder output, which would cost latency in a live call. + enc_ctx->max_b_frames = 0; + + // Note the absence of AV_CODEC_FLAG_GLOBAL_HEADER: we want the parameter + // sets repeated in-band with every keyframe, since the receiving side has + // no out-of-band way to learn them and may start decoding mid-stream. + av_opt_set(enc_ctx->priv_data, "preset", "veryfast", 0); + av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0); + + if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) < 0) + { + QString msg = QString("Could not open H.264 encoder: %1").arg(ff_errstr(ret)); + qWarning() << msg; + emit errorOccurred(msg); + avcodec_free_context(&enc_ctx); + avcodec_free_context(&dec_ctx); + return; + } } + emit accelerationInUse(hardware ? QStringLiteral("GPU encoding (VAAPI)") + : QStringLiteral("CPU encoding (libx264)")); + // Everything the camera sends is scaled to the encoder's size and re-encoded, // so this is what the peer will receive regardless of what the camera // produced. The codec name is taken from the encoder rather than written @@ -416,10 +535,16 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) emit captureStarted(QLatin1String(avcodec_get_name(enc_ctx->codec_id)), enc_ctx->width, enc_ctx->height); + // What the camera's frames are scaled into. With a hardware encoder this + // is only a staging buffer in system memory - NV12, which is what VAAPI + // takes - that gets uploaded to a GPU surface below; the encoder itself + // consumes AV_PIX_FMT_VAAPI frames and never sees this one. + AVPixelFormat sw_pix_fmt = hardware ? AV_PIX_FMT_NV12 : enc_ctx->pix_fmt; + AVFrame *enc_frame = av_frame_alloc(); if (enc_frame) { - enc_frame->format = enc_ctx->pix_fmt; + enc_frame->format = sw_pix_fmt; enc_frame->width = enc_ctx->width; enc_frame->height = enc_ctx->height; } @@ -429,6 +554,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) qWarning() << msg; emit errorOccurred(msg); if (enc_frame) av_frame_free(&enc_frame); + if (hw_frames_ref) av_buffer_unref(&hw_frames_ref); avcodec_free_context(&enc_ctx); avcodec_free_context(&dec_ctx); return; @@ -471,7 +597,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) if (!sws) { sws = sws_getContext(cam_frame->width, cam_frame->height, (AVPixelFormat)cam_frame->format, - enc_ctx->width, enc_ctx->height, enc_ctx->pix_fmt, + enc_ctx->width, enc_ctx->height, sw_pix_fmt, SWS_BILINEAR, nullptr, nullptr, nullptr); if (!sws) { @@ -497,7 +623,41 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) enc_frame->pts = pts++; - if (avcodec_send_frame(enc_ctx, enc_frame) != 0) + // What actually goes to the encoder: the staging frame itself in + // software, or a GPU surface holding a copy of it in hardware. + AVFrame *frame_to_encode = enc_frame; +#ifdef HAVE_VAAPI + AVFrame *hw_frame = nullptr; + + if (hardware) + { + hw_frame = av_frame_alloc(); + + if (!hw_frame || + av_hwframe_get_buffer(enc_ctx->hw_frames_ctx, hw_frame, 0) < 0 || + av_hwframe_transfer_data(hw_frame, enc_frame, 0) < 0) + { + if (!loggedEncodeError) + { + qWarning() << "Could not upload the frame to the GPU"; + loggedEncodeError = true; + } + if (hw_frame) av_frame_free(&hw_frame); + continue; + } + + hw_frame->pts = enc_frame->pts; + frame_to_encode = hw_frame; + } +#endif + + int send_ret = avcodec_send_frame(enc_ctx, frame_to_encode); +#ifdef HAVE_VAAPI + if (hw_frame) + av_frame_free(&hw_frame); +#endif + + if (send_ret != 0) { if (!loggedEncodeError) { @@ -520,6 +680,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) av_frame_free(&cam_frame); av_frame_free(&enc_frame); if (sws) sws_freeContext(sws); + if (hw_frames_ref) av_buffer_unref(&hw_frames_ref); avcodec_free_context(&enc_ctx); avcodec_free_context(&dec_ctx); } @@ -541,6 +702,29 @@ void VideoDevice::Play() if (dec && dec->type != AVMEDIA_TYPE_VIDEO) dec = nullptr; AVCodecContext *dec_ctx = dec ? avcodec_alloc_context3(dec) : nullptr; + +#ifdef HAVE_VAAPI + // Has to be attached before opening the decoder. Unlike encoding, there is + // no separate hardware decoder to look up: the ordinary decoder is told to + // use a VAAPI device, and get_format then picks the GPU pixel format when + // the codec and hardware between them can manage it. + if (m_useVaapi && dec_ctx && VaapiContext::isAvailable()) + { + AVBufferRef *hw_device_ref = VaapiContext::deviceRef(); + + if (hw_device_ref) + { + // The context takes ownership of this reference. + dec_ctx->hw_device_ctx = hw_device_ref; + dec_ctx->get_format = vaapi_get_format; + } + else + { + qWarning() << "No usable VAAPI device, decoding in software"; + } + } +#endif + if (!dec || !dec_ctx || avcodec_open2(dec_ctx, dec, nullptr) < 0) { QString msg = QString("Unsupported call video codec: %1").arg(m_codec); @@ -557,6 +741,11 @@ void VideoDevice::Play() AVFrame *rgb_frame = av_frame_alloc(); bool loggedDecodeError = false; bool loggedScaleError = false; + bool loggedTransferError = false; + // Whether the GPU is really doing the decoding is only knowable from a + // decoded frame, so the answer is reported once the first one arrives + // rather than guessed from what was asked for. + bool reportedAcceleration = false; while (true) { @@ -597,15 +786,74 @@ void VideoDevice::Play() while (avcodec_receive_frame(dec_ctx, dec_frame) == 0) { + bool decodedOnGpu = false; +#ifdef HAVE_VAAPI + decodedOnGpu = (dec_frame->format == AV_PIX_FMT_VAAPI); +#endif + + if (!reportedAcceleration) + { + emit accelerationInUse(decodedOnGpu ? QStringLiteral("GPU decoding (VAAPI)") + : QStringLiteral("CPU decoding")); + reportedAcceleration = true; + } + + // The frame the rest of this iteration works from. A GPU frame + // holds no pixels, so unless it is going straight to a renderer + // that can draw it, it has to be brought down into system memory + // first. + AVFrame *display_frame = dec_frame; +#ifdef HAVE_VAAPI + AVFrame *sw_frame = nullptr; + + if (decodedOnGpu && m_useVaapiRender) + { + // Zero copy: hand the surface itself over, holding a + // reference so it stays alive until the renderer is done. + AVFrame *hw_ref = av_frame_alloc(); + + if (hw_ref && av_frame_ref(hw_ref, dec_frame) == 0) + { + emit hwFrameReady(wrapAVFrame(hw_ref)); + } + else if (hw_ref) + { + av_frame_free(&hw_ref); + } + + av_frame_unref(dec_frame); + continue; + } + + if (decodedOnGpu) + { + sw_frame = av_frame_alloc(); + + if (!sw_frame || av_hwframe_transfer_data(sw_frame, dec_frame, 0) < 0) + { + if (!loggedTransferError) + { + qWarning() << "Could not read the decoded frame back from the GPU"; + loggedTransferError = true; + } + if (sw_frame) av_frame_free(&sw_frame); + av_frame_unref(dec_frame); + continue; + } + + display_frame = sw_frame; + } +#endif + // Rebuilt whenever the incoming picture format or size changes - // including the first frame, when they first become known. - if (!sws || dec_frame->width != sws_width || dec_frame->height != sws_height || - (AVPixelFormat)dec_frame->format != sws_fmt) + if (!sws || display_frame->width != sws_width || display_frame->height != sws_height || + (AVPixelFormat)display_frame->format != sws_fmt) { if (sws) sws_freeContext(sws); - sws_width = dec_frame->width; - sws_height = dec_frame->height; - sws_fmt = (AVPixelFormat)dec_frame->format; + sws_width = display_frame->width; + sws_height = display_frame->height; + sws_fmt = (AVPixelFormat)display_frame->format; sws = sws_getContext(sws_width, sws_height, sws_fmt, sws_width, sws_height, AV_PIX_FMT_RGB24, @@ -624,6 +872,9 @@ void VideoDevice::Play() loggedScaleError = true; } if (sws) { sws_freeContext(sws); sws = nullptr; } +#ifdef HAVE_VAAPI + if (sw_frame) av_frame_free(&sw_frame); +#endif av_frame_unref(dec_frame); continue; } @@ -631,11 +882,14 @@ void VideoDevice::Play() if (av_frame_make_writable(rgb_frame) < 0) { +#ifdef HAVE_VAAPI + if (sw_frame) av_frame_free(&sw_frame); +#endif av_frame_unref(dec_frame); continue; } - sws_scale(sws, dec_frame->data, dec_frame->linesize, 0, dec_frame->height, + sws_scale(sws, display_frame->data, display_frame->linesize, 0, display_frame->height, rgb_frame->data, rgb_frame->linesize); // Deep copy: the QImage outlives this iteration (it travels to @@ -645,6 +899,9 @@ void VideoDevice::Play() rgb_frame->linesize[0], QImage::Format_RGB888); emit frameReady(img.copy()); +#ifdef HAVE_VAAPI + if (sw_frame) av_frame_free(&sw_frame); +#endif av_frame_unref(dec_frame); } } diff --git a/pica-client/videodevice.h b/pica-client/videodevice.h index 1b999c1..49d86a9 100644 --- a/pica-client/videodevice.h +++ b/pica-client/videodevice.h @@ -18,6 +18,7 @@ #define VIDEODEVICE_H #include "mediadevice.h" +#include "vaapi.h" #include <QObject> #include <QString> #include <QByteArray> @@ -79,8 +80,21 @@ public: // re-encoding every frame. The codec that ends up being used is only // known once the camera has been opened, and is reported by // captureStarted() - it is not necessarily the one requested here. - Q_INVOKABLE void configureCapture(QString deviceName, int width, int height, bool preferCompressed); - Q_INVOKABLE void configurePlayback(QString codec, int width, int height); + // + // useVaapi asks for the frames to be encoded on the GPU. It is a request, + // not a guarantee: without VAAPI support built in, without usable + // hardware, or if the hardware encoder will not open, capture falls back + // to encoding in software. + Q_INVOKABLE void configureCapture(QString deviceName, int width, int height, + bool preferCompressed, bool useVaapi); + + // useVaapi decodes on the GPU. useVaapiRender additionally keeps the + // decoded frames there and emits them through hwFrameReady() instead of + // frameReady(), for a renderer that can draw a VA surface directly; it + // implies GPU decoding, since there is no surface to draw otherwise. Both + // fall back to software the same way capture does. + Q_INVOKABLE void configurePlayback(QString codec, int width, int height, + bool useVaapi, bool useVaapiRender); // Push one complete encoded frame (already reassembled from 0x77 // fragments by the caller) into the decode queue. Safe to call from any @@ -119,8 +133,21 @@ signals: // One decoded picture from the remote side, ready to be displayed. void frameReady(QImage frame); +#ifdef HAVE_VAAPI + // One decoded picture still living in GPU memory as a VA surface, for a + // renderer that can draw it without a round trip through system memory. + // Emitted instead of frameReady() when GPU rendering is in use. + void hwFrameReady(AVFramePtr frame); +#endif + void errorOccurred(QString message); + // Says in plain words which path capture or playback actually settled on, + // for the settings dialog to show. A request for GPU work can silently + // come back as software, and forwarding a compressed camera stream skips + // encoding altogether - neither should have to be guessed at. + void accelerationInUse(QString description); + public: virtual QList<MediaDeviceInfo> Enumerate(enum MediaDeviceStreamDirection dir); @@ -141,6 +168,8 @@ private: int m_width; int m_height; bool m_preferCompressed; + bool m_useVaapi; + bool m_useVaapiRender; QAtomicInt m_abort; |