pica-client: add d3d12va encoding acceleration on Windows

4 files changed, 241 insertions, 69 deletions

diff --git a/configure.ac b/configure.ac
index eea64d1..f7b48f6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -92,6 +92,16 @@ AC_ARG_ENABLE([vaapi],
     AS_HELP_STRING([--disable-vaapi],
       [Disable VAAPI accelerated video encoding, decoding and rendering]))
 
+#checking if Direct3D 12 hardware encoding is wanted. The Windows counterpart
+#of the VAAPI encoding path above - there is no libva there, but FFmpeg can
+#reach the same GPU encoders through D3D12 Video. Autodetected the same way,
+#and only ever looked for when building for Windows.
+AC_ARG_ENABLE([d3d12va],
+    AS_HELP_STRING([--enable-d3d12va],
+      [Enable Direct3D 12 accelerated video encoding, Windows only (default: autodetect)])
+    AS_HELP_STRING([--disable-d3d12va],
+      [Disable Direct3D 12 accelerated video encoding]))
+
 #checking if PulseAudio device enumeration is wanted. Only affects what the
 #settings dialog can list; see the check further down.
 AC_ARG_ENABLE([pulse],
@@ -313,6 +323,48 @@ AS_IF([test "x$PICA_VAAPI_ENABLED" = "xyes" && test "x$enable_vaapi_x11" != "xno
         [AC_MSG_WARN(["libva-x11, libX11 or libGL not found - building without X11 rendering, hardware rendering will need an EGL session"])])
     ])
 ])
+
+#check whether FFmpeg knows how to encode through Direct3D 12, which is how a
+#Windows build reaches the GPU: libva is not part of that platform, but the
+#hwcontext machinery the VAAPI path uses carries a D3D12 device type as well,
+#and libavcodec has h264_d3d12va and hevc_d3d12va encoders to go with it.
+#
+#Nothing is linked for this - libavutil loads d3d12.dll itself when a device is
+#created - so all there is to check is that the headers are new enough to name
+#the device type and the pixel format. Whether those encoders were compiled
+#into the libavcodec being built against, and whether the machine the client
+#ends up running on has a GPU that can encode, are both runtime questions,
+#answered by falling back to the software encoder.
+AS_IF([test "x$TARGET_OS" = "xwindows" && test "x$enable_d3d12va" != "xno"], [
+  AC_MSG_CHECKING([whether FFmpeg supports Direct3D 12 video encoding])
+
+  pica_save_CPPFLAGS="$CPPFLAGS"
+  CPPFLAGS="$CPPFLAGS $FFMPEG_CFLAGS"
+
+  AC_COMPILE_IFELSE(
+    [AC_LANG_PROGRAM(
+      [[#include <libavutil/hwcontext.h>
+        #include <libavutil/pixfmt.h>]],
+      [[enum AVHWDeviceType device = AV_HWDEVICE_TYPE_D3D12VA;
+        enum AVPixelFormat format = AV_PIX_FMT_D3D12;
+        (void)device; (void)format;]])],
+    [
+      PICA_D3D12VA_ENABLED=yes
+      AC_DEFINE([HAVE_D3D12VA], [1], [Define to 1 if FFmpeg can encode video through Direct3D 12])
+      AC_MSG_RESULT([yes])
+    ],
+    [
+      AC_MSG_RESULT([no])
+      AS_IF([test "x$enable_d3d12va" = "xyes"],
+        [AC_MSG_ERROR(["Direct3D 12 encoding was requested but this FFmpeg has no D3D12 hardware context"])],
+        [AC_MSG_WARN(["this FFmpeg has no D3D12 hardware context - building without Direct3D 12 encoding"])])
+    ])
+
+  CPPFLAGS="$pica_save_CPPFLAGS"
+])
+
+AS_IF([test "x$TARGET_OS" != "xwindows" && test "x$enable_d3d12va" = "xyes"],
+  [AC_MSG_ERROR(["Direct3D 12 encoding was requested but it only exists on Windows"])])
 ])
 
 AM_CONDITIONAL([PICA_VAAPI], [test "x$PICA_VAAPI_ENABLED" = "xyes"])
diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp
index 0cfd965..a768b55 100644
--- a/pica-client/dialogs/settingsdialog.cpp
+++ b/pica-client/dialogs/settingsdialog.cpp
@@ -375,8 +375,19 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
 
 	cbPreferCompressed = new QCheckBox(tr("Prefer compressed formats if provided by the camera"), this);
 
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
+	// Encoding is offered wherever there is an accelerator to encode on,
+	// named after the one this build uses; decoding and rendering below are
+	// VAAPI only, since the surfaces a D3D12 decoder produces have no way yet
+	// to reach the widget that would draw them.
 #ifdef HAVE_VAAPI
 	cbVaapiEncoding = new QCheckBox(tr("enable VAAPI-accelerated hardware encoding"), this);
+#else
+	cbVaapiEncoding = new QCheckBox(tr("enable Direct3D 12 accelerated hardware encoding"), this);
+#endif
+#endif
+
+#ifdef HAVE_VAAPI
 	cbVaapiDecoding = new QCheckBox(tr("enable VAAPI-accelerated hardware decoding"), this);
 	cbVaapiRendering = new QCheckBox(tr("enable VAAPI accelerated hardware rendering"), this);
 	// Drawing a VA surface means having decoded into one first, so rendering
@@ -418,8 +429,10 @@ SettingsDialog::SettingsDialog(QWidget *parent) :
 	videodevLayout->addWidget(videoDevRefresh);
 	videodevLayout->addLayout(videoFormatLayout);
 	videodevLayout->addWidget(cbPreferCompressed);
-#ifdef HAVE_VAAPI
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
 	videodevLayout->addWidget(cbVaapiEncoding);
+#endif
+#ifdef HAVE_VAAPI
 	videodevLayout->addWidget(cbVaapiDecoding);
 	videodevLayout->addWidget(cbVaapiRendering);
 #endif
@@ -884,7 +897,7 @@ void SettingsDialog::toggleVideoTest()
 	videoFpsTimer->start();
 
 	bool vaapiEncoding = false;
-#ifdef HAVE_VAAPI
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
 	vaapiEncoding = cbVaapiEncoding->isChecked();
 #endif
 
@@ -1647,8 +1660,10 @@ void SettingsDialog::loadSettings()
 	// actually changes - and the stored codec is often the one already showing.
 	callAudioCodecChanged();
 
-#ifdef HAVE_VAAPI
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
 	cbVaapiEncoding->setChecked(st.loadValue("video.vaapi_encoding", 0).toBool());
+#endif
+#ifdef HAVE_VAAPI
 	cbVaapiDecoding->setChecked(st.loadValue("video.vaapi_decoding", 0).toBool());
 	cbVaapiRendering->setChecked(st.loadValue("video.vaapi_rendering", 0).toBool());
 #endif
@@ -1744,8 +1759,10 @@ void SettingsDialog::storeSettings()
 		audioBitrateKbps = callAudioBitrate->value();
 
 	st.storeValue("call.audio_bitrate_kbps", QString::number(audioBitrateKbps));
-#ifdef HAVE_VAAPI
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
 	st.storeValue("video.vaapi_encoding", cbVaapiEncoding->isChecked() ? "1" : "0");
+#endif
+#ifdef HAVE_VAAPI
 	st.storeValue("video.vaapi_decoding", cbVaapiDecoding->isChecked() ? "1" : "0");
 	st.storeValue("video.vaapi_rendering", cbVaapiRendering->isChecked() ? "1" : "0");
 #endif
diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp
index 7e4976a..419d1df 100644
--- a/pica-client/videodevice.cpp
+++ b/pica-client/videodevice.cpp
@@ -57,7 +57,7 @@ extern "C" {
 #include <libavutil/mem.h>
 #include <libavutil/error.h>
 #include <libswscale/swscale.h>
-#ifdef HAVE_VAAPI
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
 #include <libavutil/hwcontext.h>
 #endif
 }
@@ -110,21 +110,28 @@ static const struct
 #endif
 
 // Which FFmpeg encoder produces each of the codecs the "Video Codec" setting
-// offers, on the GPU and in software. Two names per codec because FFmpeg names
-// its external library encoders after the library rather than after the codec,
-// and the id as well so that a build without the library named here can still
-// fall back to whatever other encoder it has for the same codec.
+// offers, on the GPU and in software. Several names per codec because FFmpeg
+// names its external library encoders after the library rather than after the
+// codec, and names a hardware one after the API that drives it - VAAPI where
+// there is libva, Direct3D 12 on Windows. The id as well, so that a build
+// without the library named here can still fall back to whatever other encoder
+// it has for the same codec.
+//
+// A null hardware name means that API has no encoder for the codec: D3D12
+// Video defines H.264, HEVC and AV1 encoding and no VP9, so a call set to VP9
+// encodes in software on Windows however the acceleration setting stands.
 static const struct
 {
 	const char *codec;
 	AVCodecID id;
 	const char *vaapiEncoder;
+	const char *d3d12vaEncoder;
 	const char *swEncoder;
 } kVideoEncoders[] =
 {
-	{ "h264", AV_CODEC_ID_H264, "h264_vaapi", "libx264"    },
-	{ "hevc", AV_CODEC_ID_HEVC, "hevc_vaapi", "libx265"    },
-	{ "vp9",  AV_CODEC_ID_VP9,  "vp9_vaapi",  "libvpx-vp9" }
+	{ "h264", AV_CODEC_ID_H264, "h264_vaapi", "h264_d3d12va", "libx264"    },
+	{ "hevc", AV_CODEC_ID_HEVC, "hevc_vaapi", "hevc_d3d12va", "libx265"    },
+	{ "vp9",  AV_CODEC_ID_VP9,  "vp9_vaapi",  nullptr,        "libvpx-vp9" }
 };
 
 static int videoEncoderIndex(const QString &codec)
@@ -1281,6 +1288,83 @@ static AVPixelFormat swscaleSourceFormat(AVPixelFormat format, bool *fullRange)
 	return format;
 }
 
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
+// Opens one of FFmpeg's hardware encoders on a device that has already been
+// created, and makes the pool of GPU surfaces it takes its input from.
+//
+// Shared by the VAAPI and the Direct3D 12 paths, which differ only in the
+// device handed in here and in the pixel format naming the memory the surfaces
+// live in - from the pool onwards it is the same hwcontext machinery, and the
+// encoder is configured exactly as the software one is below.
+//
+// Returns the opened context with the pool it draws from in *hw_frames_ref, or
+// null with nothing left allocated and the reason logged, which leaves the
+// caller to encode in software.
+static AVCodecContext *openHwEncoder(const AVCodec *enc, AVBufferRef *hw_device_ref,
+                                     AVPixelFormat hw_pix_fmt, int width, int height,
+                                     AVRational frameRate, int64_t bitrate, int gopSize,
+                                     AVBufferRef **hw_frames_ref)
+{
+	AVCodecContext *enc_ctx = avcodec_alloc_context3(enc);
+
+	*hw_frames_ref = enc_ctx ? av_hwframe_ctx_alloc(hw_device_ref) : nullptr;
+
+	if (!*hw_frames_ref)
+	{
+		qWarning() << QString("Could not create a surface pool for the %1 encoder, encoding in software")
+		              .arg(QLatin1String(enc->name));
+		if (enc_ctx) avcodec_free_context(&enc_ctx);
+		return nullptr;
+	}
+
+	// The pool the encoder draws its input surfaces from. NV12 is what these
+	// encoders take; the frames themselves live in GPU memory, which is what
+	// hw_pix_fmt denotes.
+	AVHWFramesContext *frames_ctx = (AVHWFramesContext *)(*hw_frames_ref)->data;
+	frames_ctx->format = hw_pix_fmt;
+	frames_ctx->sw_format = AV_PIX_FMT_NV12;
+	frames_ctx->width = width;
+	frames_ctx->height = height;
+	frames_ctx->initial_pool_size = 20;
+
+	int ret = av_hwframe_ctx_init(*hw_frames_ref);
+
+	if (ret < 0)
+	{
+		qWarning() << QString("Could not initialise the surface pool for the %1 encoder (%2), encoding in software")
+		              .arg(QLatin1String(enc->name), ff_errstr(ret));
+		av_buffer_unref(hw_frames_ref);
+		avcodec_free_context(&enc_ctx);
+		return nullptr;
+	}
+
+	enc_ctx->width = width;
+	enc_ctx->height = height;
+	enc_ctx->pix_fmt = hw_pix_fmt;
+	enc_ctx->time_base = av_inv_q(frameRate);
+	enc_ctx->framerate = frameRate;
+	enc_ctx->bit_rate = bitrate;
+	enc_ctx->gop_size = gopSize;
+	enc_ctx->max_b_frames = 0;
+	enc_ctx->hw_frames_ctx = av_buffer_ref(*hw_frames_ref);
+
+	// No preset/tune here: those belong to the software encoders and mean
+	// nothing to a hardware one.
+	ret = avcodec_open2(enc_ctx, enc, nullptr);
+
+	if (ret < 0)
+	{
+		qWarning() << QString("The %1 encoder would not open (%2), encoding in software")
+		              .arg(QLatin1String(enc->name), ff_errstr(ret));
+		av_buffer_unref(hw_frames_ref);
+		avcodec_free_context(&enc_ctx);
+		return nullptr;
+	}
+
+	return enc_ctx;
+}
+#endif
+
 void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 {
 	const AVRational frameRate = cameraFrameRate(in_st, m_frameRate);
@@ -1315,9 +1399,9 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 	}
 
 	// The GPU encoder is tried first when asked for, and anything that goes
-	// wrong on the way - no such encoder built into FFmpeg, no usable VAAPI
-	// device, no surface pool, the encoder refusing to open - just leaves
-	// hardware false and takes the software path below.
+	// wrong on the way - no such encoder built into FFmpeg, no usable device,
+	// no surface pool, the encoder refusing to open - just leaves hardware
+	// false and takes the software path below.
 	bool hardware = false;
 	const AVCodec *enc = nullptr;
 	AVCodecContext *enc_ctx = nullptr;
@@ -1331,46 +1415,10 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 
 		if (hw_device_ref)
 		{
-			enc_ctx = avcodec_alloc_context3(enc);
-			hw_frames_ref = enc_ctx ? av_hwframe_ctx_alloc(hw_device_ref) : nullptr;
-
-			if (hw_frames_ref)
-			{
-				// The pool the encoder draws its input surfaces from. NV12 is
-				// what VAAPI encoders take; the frames themselves live in GPU
-				// memory, which is what AV_PIX_FMT_VAAPI denotes.
-				AVHWFramesContext *frames_ctx = (AVHWFramesContext *)hw_frames_ref->data;
-				frames_ctx->format = AV_PIX_FMT_VAAPI;
-				frames_ctx->sw_format = AV_PIX_FMT_NV12;
-				frames_ctx->width = m_width;
-				frames_ctx->height = m_height;
-				frames_ctx->initial_pool_size = 20;
-
-				if (av_hwframe_ctx_init(hw_frames_ref) >= 0)
-				{
-					enc_ctx->width = m_width;
-					enc_ctx->height = m_height;
-					enc_ctx->pix_fmt = AV_PIX_FMT_VAAPI;
-					enc_ctx->time_base = av_inv_q(frameRate);
-					enc_ctx->framerate = frameRate;
-					enc_ctx->bit_rate = m_bitrate;
-					enc_ctx->gop_size = gopSize;
-					enc_ctx->max_b_frames = 0;
-					enc_ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
-
-					// No preset/tune here: those belong to the software
-					// encoders and mean nothing to a VAAPI one.
-					if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) >= 0)
-						hardware = true;
-					else
-						qWarning() << QString("The VAAPI %1 encoder would not open (%2), encoding in software")
-						              .arg(QLatin1String(kVideoEncoders[encIdx].codec), ff_errstr(ret));
-				}
-				else
-				{
-					qWarning() << "Could not create a VAAPI surface pool, encoding in software";
-				}
-			}
+			enc_ctx = openHwEncoder(enc, hw_device_ref, AV_PIX_FMT_VAAPI,
+			                        m_width, m_height, frameRate, m_bitrate, gopSize,
+			                        &hw_frames_ref);
+			hardware = enc_ctx != nullptr;
 
 			av_buffer_unref(&hw_device_ref);
 		}
@@ -1385,11 +1433,62 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 		}
 
 		if (!hardware)
-		{
-			if (enc_ctx) avcodec_free_context(&enc_ctx);
-			if (hw_frames_ref) av_buffer_unref(&hw_frames_ref);
 			enc = nullptr;
+	}
+#endif
+
+#ifdef HAVE_D3D12VA
+	// What the same request means on Windows, where the GPU is reached
+	// through D3D12 Video rather than through libva. Written as a fallback
+	// after the VAAPI attempt rather than as its alternative so that a build
+	// that somehow has both - libva has a Windows port of its own - still
+	// gets a hardware encoder if only one of the two comes up.
+	//
+	// The device is created per capture session and dropped with it, unlike
+	// the VAAPI one: nothing else in the client holds a D3D12 device, since
+	// decoding and rendering do not go through it.
+	if (!hardware && m_useVaapi)
+	{
+		const char *encoderName = kVideoEncoders[encIdx].d3d12vaEncoder;
+
+		enc = encoderName ? avcodec_find_encoder_by_name(encoderName) : nullptr;
+
+		AVBufferRef *hw_device_ref = nullptr;
+		ret = enc ? av_hwdevice_ctx_create(&hw_device_ref, AV_HWDEVICE_TYPE_D3D12VA,
+		                                   nullptr, nullptr, 0)
+		          : 0;
+
+		if (hw_device_ref)
+		{
+			enc_ctx = openHwEncoder(enc, hw_device_ref, AV_PIX_FMT_D3D12,
+			                        m_width, m_height, frameRate, m_bitrate, gopSize,
+			                        &hw_frames_ref);
+			hardware = enc_ctx != nullptr;
+
+			av_buffer_unref(&hw_device_ref);
+		}
+		else if (enc)
+		{
+			// Where the GPU or the version of Windows cannot encode at all,
+			// which is a plain answer rather than a fault: D3D12 video
+			// encoding needs Windows 11, or Windows 10 with the Agility SDK
+			// alongside the client.
+			qWarning() << QString("No usable Direct3D 12 encoding device (%1), encoding in software")
+			              .arg(ff_errstr(ret));
 		}
+		else if (encoderName)
+		{
+			qWarning() << QString("FFmpeg has no %1 encoder, encoding in software")
+			              .arg(QLatin1String(encoderName));
+		}
+		else
+		{
+			qWarning() << QString("Direct3D 12 cannot encode %1, encoding in software")
+			              .arg(QLatin1String(kVideoEncoders[encIdx].codec));
+		}
+
+		if (!hardware)
+			enc = nullptr;
 	}
 #endif
 
@@ -1451,9 +1550,12 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 	                       ? QStringLiteral("uncompressed")
 	                       : QString::fromLatin1(avcodec_get_name(in_st->codecpar->codec_id));
 
+	// Named from the encoder that was actually opened rather than written out
+	// per API: "h264_vaapi" and "hevc_d3d12va" say both which codec and which
+	// accelerator settled it, and cannot drift from what is running.
 	emit accelerationInUse(hardware
-	                       ? QString("GPU encoding (VAAPI %1) from %2")
-	                         .arg(QLatin1String(kVideoEncoders[encIdx].codec), source)
+	                       ? QString("GPU encoding (%1) from %2")
+	                         .arg(QLatin1String(enc->name), source)
 	                       : QString("CPU encoding (%1) from %2").arg(swEncoderName, source));
 
 	// Everything the camera sends is scaled to the encoder's size and re-encoded,
@@ -1470,9 +1572,9 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 	                    enc_ctx->width, enc_ctx->height, av_q2d(frameRate));
 
 	// What the camera's frames are scaled into. With a hardware encoder this
-	// is only a staging buffer in system memory - NV12, which is what VAAPI
-	// takes - that gets uploaded to a GPU surface below; the encoder itself
-	// consumes AV_PIX_FMT_VAAPI frames and never sees this one.
+	// is only a staging buffer in system memory - NV12, which is what the
+	// GPU encoders take - that gets uploaded to a GPU surface below; the
+	// encoder itself consumes surfaces and never sees this frame.
 	AVPixelFormat sw_pix_fmt = hardware ? AV_PIX_FMT_NV12 : enc_ctx->pix_fmt;
 
 	AVFrame *enc_frame = av_frame_alloc();
@@ -1576,7 +1678,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 			// What actually goes to the encoder: the staging frame itself in
 			// software, or a GPU surface holding a copy of it in hardware.
 			AVFrame *frame_to_encode = enc_frame;
-#ifdef HAVE_VAAPI
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
 			AVFrame *hw_frame = nullptr;
 
 			if (hardware)
@@ -1602,7 +1704,7 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st)
 #endif
 
 			int send_ret = avcodec_send_frame(enc_ctx, frame_to_encode);
-#ifdef HAVE_VAAPI
+#if defined(HAVE_VAAPI) || defined(HAVE_D3D12VA)
 			if (hw_frame)
 				av_frame_free(&hw_frame);
 #endif
diff --git a/pica-client/videodevice.h b/pica-client/videodevice.h
index 02be737..5c4585c 100644
--- a/pica-client/videodevice.h
+++ b/pica-client/videodevice.h
@@ -173,10 +173,11 @@ public:
 	// looking for one to forward, so choosing a codec the camera can produce
 	// itself gets that one rather than merely a compatible one.
 	//
-	// useVaapi asks for the frames to be encoded on the GPU. It is a request,
-	// not a guarantee: without VAAPI support built in, without usable
-	// hardware, or if the hardware encoder will not open, capture falls back
-	// to encoding in software.
+	// useVaapi asks for the frames to be encoded on the GPU, through whichever
+	// accelerator the build has - VAAPI where there is libva, Direct3D 12 on
+	// Windows. It is a request, not a guarantee: with no such support built
+	// in, without usable hardware, or if the hardware encoder will not open,
+	// capture falls back to encoding in software.
 	Q_INVOKABLE void configureCapture(QString deviceName, int width, int height, int frameRate,
 	                                  bool preferCompressed, QString codec, int bitrate,
 	                                  bool useVaapi);