pica-client: callwindow layout, video resize

1 files changed, 86 insertions, 25 deletions

diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp
index 3fb468c..f657706 100644
--- a/pica-client/videodevice.cpp
+++ b/pica-client/videodevice.cpp
@@ -16,6 +16,9 @@
 */
 #include "videodevice.h"
 #include "callsettings.h"
+// For the shared decision about what size a frame is drawn at - the decode loop
+// produces frames at exactly the size CallWindow expects, see VideoScaler.
+#include "videoscaler.h"
 #include "../PICA_proto.h"
 #include "../PICA_media.h"
 
@@ -587,10 +590,39 @@ VideoDevice::VideoDevice(QObject *parent)
 	  m_frameRate(kDefaultCaptureFrameRate), m_bitrate(kDefaultVideoBitrateKbps * 1000),
 	  m_preferCompressed(false),
 	  m_useVaapi(false), m_useVaapiRender(false), m_abort(0),
-	  m_maxFragmentSize(kMaxFragmentSize), m_decoderStarted(0)
+	  m_maxFragmentSize(kMaxFragmentSize), m_displaySize(0), m_decoderStarted(0)
 {
 }
 
+void VideoDevice::setDisplaySize(QSize size)
+{
+	// Packed into one atomic so that the decode loop always reads a width and
+	// a height that belong together - see m_displaySize. Sizes beyond what
+	// half an int holds are not real displays, and clamping them is enough:
+	// the frame is fitted inside this area, so an area larger than the frame
+	// simply means no resizing.
+	int width = qBound(0, size.width(), 0xFFFF);
+	int height = qBound(0, size.height(), 0xFFFF);
+
+	if (width == 0 || height == 0)
+	{
+		m_displaySize.storeRelaxed(0);
+		return;
+	}
+
+	m_displaySize.storeRelaxed((width << 16) | height);
+}
+
+QSize VideoDevice::displaySize() const
+{
+	int packed = m_displaySize.loadRelaxed();
+
+	if (packed == 0)
+		return QSize();
+
+	return QSize((packed >> 16) & 0xFFFF, packed & 0xFFFF);
+}
+
 void VideoDevice::setMaxFragmentSize(int size)
 {
 	if (size <= 0 || size > kMaxFragmentSize)
@@ -1356,8 +1388,11 @@ void VideoDevice::Play()
 	SwsContext *sws = nullptr;
 	int sws_width = 0, sws_height = 0;
 	AVPixelFormat sws_fmt = AV_PIX_FMT_NONE;
+	// Size the scaler is producing, which is the size of the area the window
+	// reported through setDisplaySize() with the frame fitted into it - not
+	// the size of the decoded frame.
+	int sws_dst_width = 0, sws_dst_height = 0;
 	AVFrame *dec_frame = av_frame_alloc();
-	AVFrame *rgb_frame = av_frame_alloc();
 	bool loggedDecodeError = false;
 	bool loggedScaleError = false;
 	bool loggedTransferError = false;
@@ -1470,33 +1505,54 @@ void VideoDevice::Play()
 			}
 #endif
 
-			// Rebuilt whenever the incoming picture format or size changes -
-			// including the first frame, when they first become known.
-			if (!sws || display_frame->width != sws_width || display_frame->height != sws_height ||
-			    (AVPixelFormat)display_frame->format != sws_fmt)
+			// The picture is converted straight to the size it is going to
+			// be drawn at, rather than to its own size for the window to
+			// resize afterwards. The resize costs almost nothing here,
+			// inside a colour conversion that has to run regardless, and
+			// nothing is converted only to be discarded when the window is
+			// smaller than the frame - between 1.1 and 4.6 times cheaper
+			// over the whole pipeline, see tests/test_videoscaler.
+			//
+			// Read once per frame: the window reports a new area on every
+			// resize, and mid-call resolution changes move the frame size
+			// underneath it as well.
+			QSize frameSize(display_frame->width, display_frame->height);
+			QSize displayArea = displaySize();
+			QSize target = displayArea.isEmpty() ? frameSize
+			                                     : VideoScaler::fitted(frameSize, displayArea);
+
+			if (target.isEmpty())
+				target = frameSize;
+
+			// Rebuilt whenever the incoming picture format or size changes,
+			// or the window is resized - including on the first frame, when
+			// they first become known.
+			if (!sws || frameSize.width() != sws_width || frameSize.height() != sws_height ||
+			    (AVPixelFormat)display_frame->format != sws_fmt ||
+			    target.width() != sws_dst_width || target.height() != sws_dst_height)
 			{
 				if (sws) sws_freeContext(sws);
-				sws_width = display_frame->width;
-				sws_height = display_frame->height;
+				sws_width = frameSize.width();
+				sws_height = frameSize.height();
 				sws_fmt = (AVPixelFormat)display_frame->format;
+				sws_dst_width = target.width();
+				sws_dst_height = target.height();
 
+				// AV_PIX_FMT_0RGB32 is laid out exactly as
+				// QImage::Format_RGB32 on either endianness, so the
+				// window gets a 32 bit image it can draw without a
+				// conversion of its own - which Format_RGB888 needed.
 				sws = sws_getContext(sws_width, sws_height, sws_fmt,
-				                     sws_width, sws_height, AV_PIX_FMT_RGB24,
+				                     sws_dst_width, sws_dst_height, AV_PIX_FMT_0RGB32,
 				                     SWS_BILINEAR, nullptr, nullptr, nullptr);
 
-				av_frame_unref(rgb_frame);
-				rgb_frame->format = AV_PIX_FMT_RGB24;
-				rgb_frame->width = sws_width;
-				rgb_frame->height = sws_height;
-
-				if (!sws || av_frame_get_buffer(rgb_frame, 0) < 0)
+				if (!sws)
 				{
 					if (!loggedScaleError)
 					{
 						qWarning() << "Could not set up playback scaler";
 						loggedScaleError = true;
 					}
-					if (sws) { sws_freeContext(sws); sws = nullptr; }
 #ifdef HAVE_VAAPI
 					if (sw_frame) av_frame_free(&sw_frame);
 #endif
@@ -1505,7 +1561,15 @@ void VideoDevice::Play()
 				}
 			}
 
-			if (av_frame_make_writable(rgb_frame) < 0)
+			// Allocated per frame rather than reused, which is what lets it
+			// go to the GUI thread without being copied: the queued signal
+			// takes a reference to this buffer and the image frees itself
+			// once the window is done with it. A single reused buffer would
+			// have to be deep copied instead, since the next frame would
+			// overwrite it while the window was still drawing.
+			QImage img(target, QImage::Format_RGB32);
+
+			if (img.isNull())
 			{
 #ifdef HAVE_VAAPI
 				if (sw_frame) av_frame_free(&sw_frame);
@@ -1514,15 +1578,13 @@ void VideoDevice::Play()
 				continue;
 			}
 
+			uint8_t *dst_data[4] = { img.bits(), nullptr, nullptr, nullptr };
+			int dst_linesize[4] = { (int)img.bytesPerLine(), 0, 0, 0 };
+
 			sws_scale(sws, display_frame->data, display_frame->linesize, 0, display_frame->height,
-			          rgb_frame->data, rgb_frame->linesize);
+			          dst_data, dst_linesize);
 
-			// Deep copy: the QImage outlives this iteration (it travels to
-			// the GUI thread through a queued signal), while rgb_frame's
-			// buffer gets reused by the next scale.
-			QImage img(rgb_frame->data[0], sws_width, sws_height,
-			           rgb_frame->linesize[0], QImage::Format_RGB888);
-			emit frameReady(img.copy());
+			emit frameReady(img);
 
 #ifdef HAVE_VAAPI
 			if (sw_frame) av_frame_free(&sw_frame);
@@ -1532,7 +1594,6 @@ void VideoDevice::Play()
 	}
 
 	av_frame_free(&dec_frame);
-	av_frame_free(&rgb_frame);
 	if (sws) sws_freeContext(sws);
 	avcodec_free_context(&dec_ctx);
 }