diff options
Diffstat (limited to 'pica-client')
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 21 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.h | 7 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 125 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.h | 30 | ||||
| -rw-r--r-- | pica-client/videodevice.cpp | 505 | ||||
| -rw-r--r-- | pica-client/videodevice.h | 66 | ||||
| -rw-r--r-- | pica-client/videodshow.cpp | 25 |
7 files changed, 650 insertions, 129 deletions
diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index a6da3f5..601f3e4 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -27,7 +27,8 @@ AudioVideoCallController::AudioVideoCallController(QObject *parent) : QObject(parent), callwindow(0), ringdevice(0), is_active(false), m_audioSeq(0), - m_videoSeq(0), m_videoFrameTimestamp(0), m_videoFrameStarted(false) + m_videoSeq(0), m_videoFrameTimestamp(0), m_videoFrameStarted(false), + m_requestedFrameRate(0) { connect(skynet, SIGNAL(IncomingCall(QByteArray)), this, SLOT(call_from(QByteArray))); connect(skynet, SIGNAL(CallFailed(QByteArray,QString)), this, SLOT(call_failed(QByteArray,QString))); @@ -78,7 +79,7 @@ AudioVideoCallController::AudioVideoCallController(QObject *parent) // the audio devices above. cam = new VideoDevice(); cam->moveToThread(&cam_thread); - connect(cam, SIGNAL(captureStarted(QString,int,int)), this, SLOT(video_capture_started(QString,int,int))); + connect(cam, SIGNAL(captureStarted(QString,int,int,double)), this, SLOT(video_capture_started(QString,int,int,double))); connect(cam, SIGNAL(packetReady(QByteArray,bool)), this, SLOT(send_video_packet(QByteArray,bool))); cam_thread.start(); @@ -272,6 +273,10 @@ void AudioVideoCallController::startVideoPipeline() bool preferCompressed = st.loadValue("video.prefer_compressed", 0).toBool(); bool vaapiEncoding = st.loadValue("video.vaapi_encoding", 0).toBool(); + // Kept so that video_capture_started() can tell whether the camera + // honoured it, which is not something the camera reports. + m_requestedFrameRate = cs.captureFrameRate; + // The peer is told what we are sending only once the camera is open and // the format is settled - with a compressed camera stream being forwarded // as-is, the codec is the camera's choice, not ours (see @@ -502,11 +507,21 @@ void AudioVideoCallController::incoming_audio_packet(QByteArray peer_id, quint16 output->enqueuePacket(seq_num, data); } -void AudioVideoCallController::video_capture_started(QString codec, int width, int height) +void AudioVideoCallController::video_capture_started(QString codec, int width, int height, + double frameRate) { if (!is_active) return; + // Logged rather than announced: the 0x74 message has no field for it, and + // the peer does not need one - every frame carries its own timestamp. It + // is worth a line in the log all the same, since a camera that quietly + // substituted a lower rate than was asked for is otherwise invisible from + // the other end of a complaint about choppy video. + if (frameRate > 0.0 && frameRate < m_requestedFrameRate - 0.5) + qWarning() << QString("Camera delivering %1 fps, not the %2 that was requested") + .arg(frameRate, 0, 'f', 1).arg(m_requestedFrameRate); + // Announced only now, because what the camera turned out to deliver // decides it: a compressed camera stream is forwarded in the camera's own // codec and at the camera's own size, rather than in ours. diff --git a/pica-client/audiovideocallcontroller.h b/pica-client/audiovideocallcontroller.h index 079ebc8..59d130b 100644 --- a/pica-client/audiovideocallcontroller.h +++ b/pica-client/audiovideocallcontroller.h @@ -95,6 +95,11 @@ private: // Reassembly of incoming 0x77 fragments into whole encoded frames. VideoFrameAssembler m_videoAssembler; + // The capture frame rate this call asked the camera for, so that + // video_capture_started() can say when the camera delivered less. Zero + // before the video pipeline has been started. + int m_requestedFrameRate; + void playEarpieceTone(void (TonePlayer::*tone)()); void playRingTone(); void stopTones(); @@ -117,7 +122,7 @@ private slots: void capture_clipping(double pinnedPercent, double peakDb); void incoming_audio_packet(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data); - void video_capture_started(QString codec, int width, int height); + void video_capture_started(QString codec, int width, int height, double frameRate); void send_video_packet(QByteArray data, bool is_last_fragment); void incoming_video_params(QByteArray peer_id, QString codec, quint16 width, quint16 height); void incoming_video_packet(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data); diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index 8205c35..0cfd965 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -100,6 +100,11 @@ SettingsDialog::SettingsDialog(QWidget *parent) : // before the decoder exists. testDecoder = nullptr; videoTestRunning = false; + videoTestGpuRendering = false; + videoCapturedFrames = 0; + videoDisplayedFrames = 0; + videoCaptureFps = 0.0; + videoDisplayFps = 0.0; QVBoxLayout *settingsLayout = new QVBoxLayout(); @@ -396,7 +401,10 @@ SettingsDialog::SettingsDialog(QWidget *parent) : videoPreview->installEventFilter(this); videoTestStatus = new QLabel(this); - videoTestStatus->setAlignment(Qt::AlignCenter); + // Left aligned, not centred: this line grows and shrinks as the frame + // rate is measured and re-measured, and centring it makes the whole text + // jump sideways every time it changes length. + videoTestStatus->setAlignment(Qt::AlignLeft | Qt::AlignVCenter); #ifdef HAVE_VAAPI videoPreviewGpu = VaapiRenderWidget::create(this); @@ -427,6 +435,12 @@ SettingsDialog::SettingsDialog(QWidget *parent) : videodevtab->setLayout(videodevLayout); + // Recomputes the frame rates in the status line once a second, which is + // long enough for the number to sit still and be read. + videoFpsTimer = new QTimer(this); + videoFpsTimer->setInterval(1000); + connect(videoFpsTimer, SIGNAL(timeout()), this, SLOT(videoFpsTick())); + // Capture+encode and decode ends of the test pipeline, each blocking its // own thread while running, exactly as they do during a call. They stay // idle until the Test button is pressed. videoTestRunning is already false @@ -435,7 +449,7 @@ SettingsDialog::SettingsDialog(QWidget *parent) : testCam = new VideoDevice(); testCam->moveToThread(&testCamThread); - connect(testCam, SIGNAL(captureStarted(QString,int,int)), this, SLOT(videoTestCaptureStarted(QString,int,int))); + connect(testCam, SIGNAL(captureStarted(QString,int,int,double)), this, SLOT(videoTestCaptureStarted(QString,int,int,double))); connect(testCam, SIGNAL(packetReady(QByteArray,bool)), this, SLOT(videoTestFragment(QByteArray,bool))); connect(testCam, SIGNAL(errorOccurred(QString)), this, SLOT(videoTestError(QString))); connect(testCam, SIGNAL(accelerationInUse(QString)), this, SLOT(videoTestPath(QString))); @@ -861,6 +875,14 @@ void SettingsDialog::toggleVideoTest() // at the size the preview wants rather than the camera's own. reportVideoPreviewArea(); + videoTestGpuRendering = false; + videoCapturedFrames = 0; + videoDisplayedFrames = 0; + videoCaptureFps = 0.0; + videoDisplayFps = 0.0; + videoFpsElapsed.start(); + videoFpsTimer->start(); + bool vaapiEncoding = false; #ifdef HAVE_VAAPI vaapiEncoding = cbVaapiEncoding->isChecked(); @@ -886,14 +908,25 @@ void SettingsDialog::toggleVideoTest() // where the peer's 0x75 message arrives before its first video packet. } -void SettingsDialog::videoTestCaptureStarted(QString codec, int width, int height) +void SettingsDialog::videoTestCaptureStarted(QString codec, int width, int height, + double frameRate) { if (!videoTestRunning) return; // Says which path the test is exercising: the camera's own compressed // stream forwarded untouched, or frames decoded and re-encoded here. - videoTestFormat = tr("%1 %2x%3").arg(codec).arg(width).arg(height); + // + // The rate is the one the camera settled on, and is shown whether or not + // it is the one that was asked for - a camera substituting a lower rate + // does so silently, so seeing it here is the whole point. It is not the + // same figure as the measured rate in the status line: this is what the + // camera agreed to, that is what actually arrived, and the two differing + // means something is dropping frames rather than never producing them. + videoTestFormat = frameRate > 0.0 + ? tr("%1 %2x%3 @%4").arg(codec).arg(width).arg(height) + .arg(frameRate, 0, 'f', frameRate < 10.0 ? 1 : 0) + : tr("%1 %2x%3").arg(codec).arg(width).arg(height); videoPreview->setText(tr("Capturing %1...").arg(videoTestFormat)); bool vaapiDecoding = false; @@ -938,6 +971,10 @@ void SettingsDialog::stopVideoTest() videoTestRunning = false; videoTestFormat.clear(); videoTestPathReport.clear(); + videoTestGpuRendering = false; + videoFpsTimer->stop(); + videoCaptureFps = 0.0; + videoDisplayFps = 0.0; btVideoTest->setText(tr("Test 📷")); videoPreview->clear(); videoTestStatus->clear(); @@ -972,8 +1009,15 @@ void SettingsDialog::videoTestFragment(QByteArray data, bool is_last_fragment) testVideoSeq = (testVideoSeq + 1) & VideoFrameAssembler::SeqNumMask; if (is_last_fragment) + { seq |= VideoFrameAssembler::LastFragmentFlag; + // One encoded frame has just been completed, which in steady state is + // one frame off the camera. Counted here rather than at the camera + // because this is also what a call would be putting on the wire. + videoCapturedFrames++; + } + QByteArray frame = testAssembler.addFragment(seq, 0, data); if (frame.isEmpty()) @@ -988,16 +1032,72 @@ void SettingsDialog::videoTestFrame(QImage frame) if (!videoTestRunning || frame.isNull()) return; + videoDisplayedFrames++; + // The preview itself is showing pixels now, so what format came off the // camera and which of the GPU or CPU did the work goes underneath it. - videoTestStatus->setText(videoTestPathReport.isEmpty() - ? videoTestFormat - : tr("%1 - %2").arg(videoTestFormat, videoTestPathReport)); + updateVideoTestStatus(); lastTestFrame = frame; updateVideoPreview(); } +void SettingsDialog::videoFpsTick() +{ + const qint64 elapsed = videoFpsElapsed.restart(); + + if (elapsed <= 0) + return; + + videoCaptureFps = videoCapturedFrames * 1000.0 / elapsed; + videoDisplayFps = videoDisplayedFrames * 1000.0 / elapsed; + videoCapturedFrames = 0; + videoDisplayedFrames = 0; + + updateVideoTestStatus(); +} + +void SettingsDialog::updateVideoTestStatus() +{ + if (!videoTestRunning) + return; + + QString text = videoTestFormat; + + if (!videoTestPathReport.isEmpty()) + text = tr("%1 - %2").arg(text, videoTestPathReport); + + if (videoTestGpuRendering) + text += tr(", GPU rendering"); + + // Nothing to say until the first second is up and a rate has been + // measured, and saying "0.0 fps" in the meantime would read as a fault. + if (videoCaptureFps > 0.0 || videoDisplayFps > 0.0) + { + const int asked = selectedVideoFrameRate(); + + // The two rates are usually the same, and printing both every time + // would bury the case that matters. They part company when the + // pipeline cannot keep up with the camera - frames are then dropped + // between the two, see VideoDevice::enqueueFrame(). + if (qAbs(videoCaptureFps - videoDisplayFps) > 1.0) + text += tr(", %1 fps captured, %2 shown") + .arg(videoCaptureFps, 0, 'f', 1).arg(videoDisplayFps, 0, 'f', 1); + else + text += tr(", %1 fps").arg(videoCaptureFps, 0, 'f', 1); + + // A camera asked for more than it can deliver is not obliged to say + // so - it just sends fewer frames - so the shortfall is spelled out + // rather than left for the user to notice against the setting they + // chose. Half a frame of slack, since a camera holding its rate + // still reads a little under on any one second. + if (asked > 0 && videoCaptureFps < asked - 0.5) + text += tr(" of %1 requested").arg(asked); + } + + videoTestStatus->setText(text); +} + void SettingsDialog::updateVideoPreview() { // Only while the test is running: with it stopped the preview is showing @@ -1073,9 +1173,9 @@ void SettingsDialog::videoTestHwFrame(AVFramePtr frame) testDecoder->setDisplaySize(QSize()); } - videoTestStatus->setText(videoTestPathReport.isEmpty() - ? videoTestFormat - : tr("%1 - %2, GPU rendering").arg(videoTestFormat, videoTestPathReport)); + videoDisplayedFrames++; + videoTestGpuRendering = true; + updateVideoTestStatus(); videoPreviewGpu->setFrame(frame); } @@ -1092,6 +1192,11 @@ void SettingsDialog::videoTestRenderFailed(QString message) videoPreview->show(); videoPreview->setText(message); + // The status line must stop claiming GPU rendering; frames are about to + // start coming back through system memory instead. + videoTestGpuRendering = false; + updateVideoTestStatus(); + // Frames are about to start arriving in system memory, so the decoder needs // the size to produce them at. Sent explicitly rather than left to the // label's resize, which does not fire if showing it did not change its diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h index cd0e6b7..357edac 100644 --- a/pica-client/dialogs/settingsdialog.h +++ b/pica-client/dialogs/settingsdialog.h @@ -222,8 +222,31 @@ private: // is in videoscaler.h. VideoScaler videoTestScaler; + // Measured frame rates for the status line. Two of them, because they + // answer different questions: whether the camera is actually delivering + // the rate it was asked for, and whether the rest of the pipeline keeps up + // with what it delivers. A camera asked for more than it can manage is not + // told so - it quietly delivers fewer frames instead - so the only way to + // know is to count them. + QTimer *videoFpsTimer; + QElapsedTimer videoFpsElapsed; + int videoCapturedFrames; + int videoDisplayedFrames; + double videoCaptureFps; + double videoDisplayFps; + // Whether the preview is being drawn straight from GPU memory, which the + // status line says and which only the frames themselves reveal. + bool videoTestGpuRendering; + void stopVideoTest(); + // Builds the line under the preview out of everything known about the + // running test - format, which end did the encoding and decoding, whether + // the preview is drawn on the GPU, and the measured frame rates. One place + // for it, since the frame rates change it once a second while frames + // arriving change the rest of it. + void updateVideoTestStatus(); + // Draws lastTestFrame at the size the preview is now. void updateVideoPreview(); @@ -281,10 +304,15 @@ private slots: void audioTestPath(QString description); void toggleVideoTest(); - void videoTestCaptureStarted(QString codec, int width, int height); + void videoTestCaptureStarted(QString codec, int width, int height, double frameRate); void videoTestPath(QString description); void videoTestFragment(QByteArray data, bool is_last_fragment); void videoTestFrame(QImage frame); + // Turns the frames counted since the last tick into the rates the status + // line shows. On a timer rather than computed as frames arrive, so that a + // pipeline that has stalled reads as 0 fps instead of staying at whatever + // it managed last. + void videoFpsTick(); #ifdef HAVE_VAAPI void videoTestHwFrame(AVFramePtr frame); void videoTestRenderFailed(QString message); diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp index f657706..7e4976a 100644 --- a/pica-client/videodevice.cpp +++ b/pica-client/videodevice.cpp @@ -400,6 +400,78 @@ static QList<int> v4l2FrameRates(int fd, unsigned int pixelformat, int width, in } #endif +void pica_merge_capture_rates(VideoCaptureFormat &entry, const QString &codec, + const QList<int> &rates) +{ + int existing = -1; + + for (int p = 0; p < entry.pixelFormats.size(); p++) + { + if (entry.pixelFormats.at(p).codec == codec) + { + existing = p; + break; + } + } + + // Two of a platform's pixel formats can carry the same codec name - v4l2 + // reports MJPEG and plain JPEG separately and both come out as "mjpeg" - + // so a format already seen is added to rather than started again. + if (existing < 0) + { + VideoPixelFormat pf; + + pf.codec = codec; + entry.pixelFormats << pf; + existing = entry.pixelFormats.size() - 1; + } + + VideoPixelFormat &pf = entry.pixelFormats[existing]; + + for (int r = 0; r < rates.size(); r++) + { + if (!pf.frameRates.contains(rates.at(r))) + pf.frameRates << rates.at(r); + + if (!entry.frameRates.contains(rates.at(r))) + entry.frameRates << rates.at(r); + } +} + +void pica_finish_capture_format(VideoCaptureFormat &entry) +{ + std::sort(entry.frameRates.begin(), entry.frameRates.end(), std::greater<int>()); + + // Uncompressed first, then the compressed formats in whatever order the + // platform put compressedFormats in, rather than the order the camera + // happened to list them. + QList<VideoPixelFormat> ordered; + + for (int p = 0; p < entry.pixelFormats.size(); p++) + { + if (entry.pixelFormats.at(p).codec.isEmpty()) + ordered << entry.pixelFormats.at(p); + } + + for (int c = 0; c < entry.compressedFormats.size(); c++) + { + for (int p = 0; p < entry.pixelFormats.size(); p++) + { + if (entry.pixelFormats.at(p).codec == entry.compressedFormats.at(c)) + { + ordered << entry.pixelFormats.at(p); + break; + } + } + } + + for (int p = 0; p < ordered.size(); p++) + std::sort(ordered[p].frameRates.begin(), ordered[p].frameRates.end(), + std::greater<int>()); + + entry.pixelFormats = ordered; +} + QList<VideoCaptureFormat> VideoDevice::CaptureFormats(const QString &device) { QList<VideoCaptureFormat> result; @@ -410,9 +482,10 @@ QList<VideoCaptureFormat> VideoDevice::CaptureFormats(const QString &device) return result; // Keyed on the size, because a camera reports its sizes once per pixel - // format and the same size usually comes back several times over. What - // differs between those is the format and the frame rates available in - // it, so both are merged into the one entry for the size. + // format and the same size usually comes back several times over. Each + // format it comes back in becomes an entry in pixelFormats, keeping the + // rates that format can manage with the format itself - see + // VideoPixelFormat for why that association matters. QMap<QPair<int, int>, VideoCaptureFormat> bySize; struct v4l2_fmtdesc fmt; @@ -447,11 +520,11 @@ QList<VideoCaptureFormat> VideoDevice::CaptureFormats(const QString &device) if (!codec.isEmpty() && !entry.compressedFormats.contains(codec)) entry.compressedFormats << codec; - QList<int> rates = v4l2FrameRates(fd, fmt.pixelformat, entry.width, entry.height); - - for (int r = 0; r < rates.size(); r++) - if (!entry.frameRates.contains(rates.at(r))) - entry.frameRates << rates.at(r); + // Kept against the format it belongs to - MJPEG at 1280x720 and + // uncompressed at 1280x720 are not the same offer - and merged + // into the size's own list as well. + pica_merge_capture_rates(entry, codec, + v4l2FrameRates(fd, fmt.pixelformat, entry.width, entry.height)); } } @@ -476,7 +549,7 @@ QList<VideoCaptureFormat> VideoDevice::CaptureFormats(const QString &device) entry.compressedFormats = ordered; - std::sort(entry.frameRates.begin(), entry.frameRates.end(), std::greater<int>()); + pica_finish_capture_format(entry); result << entry; } @@ -714,15 +787,211 @@ static QByteArray cameraOpenUrl(const QString &device) #endif } +// Compressed formats in the order to prefer them as something to decode and +// re-encode, which is not the order to prefer them for forwarding untouched - +// that one is in kCompressedFormatPreference, and ranks by bytes on the wire. +// +// Here MJPEG comes first instead of last. Every frame is a standalone JPEG, so +// there are no reference frames to track and no reordering delay, it is the +// cheapest of the three to decode, and it is the one format essentially every +// UVC camera offers. Decoding a camera's H.264 only to re-encode it costs more +// and adds the decoder's delay for nothing, since the result is re-encoded to +// the configured codec and bitrate either way. +static const char * const kTranscodeSourcePreference[] = { "mjpeg", "h264", "hevc" }; + +// One way of opening the camera: which pixel format to ask the driver for, what +// to do with the packets that come back, and how much of the request to insist +// on. Built by buildCaptureAttempts() and tried in order. +struct CaptureAttempt +{ + // v4l2 input_format / dshow codec id to ask for. Empty leaves the choice + // to the camera, which means its default - uncompressed, on every camera + // seen so far. + QString inputFormat; + + // Forward the camera's packets as they arrive instead of decoding and + // re-encoding them. Only ever set for a compressed inputFormat, and only + // when the "prefer compressed formats" setting asked for it: a compressed + // format is also worth asking for purely to get the frame rate, which is + // what the same format with this false means. + bool passthrough; + + bool constrainSize; + bool constrainRate; + + // Why this attempt is in the list, for the log when it fails. + QString reason; +}; + +// Whether a camera has said this format can manage this frame rate. +// +// An empty rate list means the camera would not say, which is not the same as +// saying no - those formats are treated as capable and left to prove it at open +// time, which is how this worked before any of it was enumerated. +static bool formatHasRate(const VideoPixelFormat &format, int frameRate) +{ + return format.frameRates.isEmpty() || frameRate <= 0 || + format.frameRates.contains(frameRate); +} + +// The orders to try opening the camera in, best first. +// +// The point of consulting the enumerated formats here is that a camera asked +// for a frame rate a format cannot carry does not fail - it opens and then +// delivers fewer frames, silently. USB 2.0 does not have the bandwidth for +// uncompressed video at 30fps much above VGA, so asking for the camera's +// default format at 1280x720@30 gets 10fps and no indication of it. Asking for +// MJPEG at the same size gets 30, and decoding it costs far less than the +// missing two thirds of the frames. +// +// Hence the order: forward a compressed stream if that was asked for, then +// uncompressed if it can carry the rate - no decode, no second generation of +// lossy compression - and then compressed as a source to decode and re-encode, +// which is what lifts the frame rate above what the bus will carry raw. +// +// Everything above only considers formats the camera has said can do the job. +// The tail then repeats the whole list with the rate and then the size dropped, +// because listing a format and delivering it at a given size and rate are +// separate claims and cameras routinely honour the first without the others. +// Whatever eventually opens is what gets announced to the peer, so relaxing +// these changes what is sent, not what is claimed. +static QList<CaptureAttempt> buildCaptureAttempts(const QString &device, int width, int height, + int frameRate, bool preferCompressed, + const QString &configuredCodec) +{ + QList<CaptureAttempt> attempts; + QStringList passthroughOrder; + + if (preferCompressed) + { + passthroughOrder = VideoDevice::CompressedFormats(device); + + // The configured codec first when the camera can produce it itself: + // forwarding that one means the peer receives exactly what was asked + // for, rather than whichever compressed format the camera happened to + // rank highest. The rest keep their preference order behind it, since + // forwarding any of them still beats decoding and re-encoding. + if (passthroughOrder.removeAll(configuredCodec) > 0) + passthroughOrder.prepend(configuredCodec); + } + + // What the camera says it can do at this size. Absent - an unsupported + // platform, or a camera that would not be enumerated - leaves every + // candidate below looking capable, which is the behaviour from before any + // of this was known. + VideoCaptureFormat atSize; + + atSize.width = 0; + atSize.height = 0; + + const QList<VideoCaptureFormat> formats = VideoDevice::CaptureFormats(device); + + for (int i = 0; i < formats.size(); i++) + { + if (formats.at(i).width == width && formats.at(i).height == height) + { + atSize = formats.at(i); + break; + } + } + + // Indexed by codec name, empty string being the uncompressed one. A format + // the camera did not list at this size is absent, and absent is not + // capable - unless nothing was enumerated at all, below. + const bool enumerated = !atSize.pixelFormats.isEmpty(); + + for (int i = 0; i < passthroughOrder.size(); i++) + { + const QString codec = passthroughOrder.at(i); + bool capable = !enumerated; + + for (int p = 0; p < atSize.pixelFormats.size(); p++) + { + if (atSize.pixelFormats.at(p).codec == codec) + { + capable = formatHasRate(atSize.pixelFormats.at(p), frameRate); + break; + } + } + + if (capable) + attempts << CaptureAttempt{ codec, true, true, true, + QStringLiteral("forwarded unchanged") }; + } + + bool uncompressedCapable = !enumerated; + + for (int p = 0; p < atSize.pixelFormats.size(); p++) + { + if (atSize.pixelFormats.at(p).codec.isEmpty()) + { + uncompressedCapable = formatHasRate(atSize.pixelFormats.at(p), frameRate); + break; + } + } + + if (uncompressedCapable) + attempts << CaptureAttempt{ QString(), false, true, true, + QStringLiteral("uncompressed, re-encoded here") }; + + for (unsigned int i = 0; i < sizeof(kTranscodeSourcePreference) / sizeof(kTranscodeSourcePreference[0]); i++) + { + const QString codec = QLatin1String(kTranscodeSourcePreference[i]); + + for (int p = 0; p < atSize.pixelFormats.size(); p++) + { + if (atSize.pixelFormats.at(p).codec == codec && + formatHasRate(atSize.pixelFormats.at(p), frameRate)) + { + attempts << CaptureAttempt{ codec, false, true, true, + QStringLiteral("compressed for the frame rate, re-encoded here") }; + break; + } + } + } + + // The loosened tail. Same candidates, in the same order, with the rate + // given up and then the size as well - and without consulting what the + // camera claimed, since by this point its claims have been wrong. + QStringList tail = passthroughOrder; + QList<bool> tailPassthrough; + + for (int i = 0; i < passthroughOrder.size(); i++) + tailPassthrough << true; + + tail << QString(); + tailPassthrough << false; + + for (unsigned int i = 0; i < sizeof(kTranscodeSourcePreference) / sizeof(kTranscodeSourcePreference[0]); i++) + { + const QString codec = QLatin1String(kTranscodeSourcePreference[i]); + + if (!atSize.compressedFormats.contains(codec)) + continue; + + tail << codec; + tailPassthrough << false; + } + + for (int i = 0; i < tail.size(); i++) + attempts << CaptureAttempt{ tail.at(i), tailPassthrough.at(i), true, false, + QStringLiteral("any rate the camera will give") }; + + attempts << CaptureAttempt{ QString(), false, false, false, + QStringLiteral("anything the camera will give") }; + + return attempts; +} + // One attempt at opening the camera. // -// passthroughCodec empty asks for whatever the camera gives; a width of zero +// inputFormat empty asks for whatever the camera gives; a width of zero // drops the size and frame rate constraints as well. *ifmt_ctx is left null on // failure - avformat_open_input() frees and clears it - so the caller can just // try again. static int openCamera(const AVInputFormat *ifmt, const QByteArray &url, int width, int height, int framerate, - const QString &passthroughCodec, AVFormatContext **ifmt_ctx) + const QString &inputFormat, AVFormatContext **ifmt_ctx) { AVDictionary *opts = nullptr; @@ -732,7 +1001,7 @@ static int openCamera(const AVInputFormat *ifmt, const QByteArray &url, if (framerate > 0) av_dict_set(&opts, "framerate", QString::number(framerate).toUtf8().constData(), 0); - if (!passthroughCodec.isEmpty()) + if (!inputFormat.isEmpty()) { #ifdef Q_OS_WIN // dshow has no input_format option. It picks the compressed stream @@ -741,7 +1010,7 @@ static int openCamera(const AVInputFormat *ifmt, const QByteArray &url, // has to be set on a context allocated here, since // avformat_open_input() would otherwise make one only after the point // where the demuxer reads it. - const AVCodec *dec = avcodec_find_decoder_by_name(passthroughCodec.toUtf8().constData()); + const AVCodec *dec = avcodec_find_decoder_by_name(inputFormat.toUtf8().constData()); if (!dec) { @@ -759,7 +1028,7 @@ static int openCamera(const AVInputFormat *ifmt, const QByteArray &url, (*ifmt_ctx)->video_codec_id = dec->id; #else - av_dict_set(&opts, "input_format", passthroughCodec.toUtf8().constData(), 0); + av_dict_set(&opts, "input_format", inputFormat.toUtf8().constData(), 0); #endif } @@ -790,88 +1059,51 @@ void VideoDevice::Capture() return; } - // Ask the camera for a compressed stream when told to and it offers one: - // its packets can then go straight onto the network, with no decoding and - // re-encoding in between. The formats are queried rather than guessed, so - // these are ones the camera has already said it can deliver, best first. - // The empty string on the end is the fallback of taking whatever comes and - // re-encoding it. - QStringList attempts; - - if (m_preferCompressed) - { - attempts = CompressedFormats(m_deviceName); - - // The configured codec first when the camera can produce it itself: - // forwarding that one means the peer receives exactly what was asked - // for, rather than whichever compressed format the camera happened to - // rank highest. The rest keep their preference order behind it, since - // forwarding any of them still beats decoding and re-encoding. - if (attempts.removeAll(m_codec) > 0) - attempts.prepend(m_codec); - } - - attempts << QString(); - - // Listing a format and being able to produce it at a given size and frame - // rate are three separate claims, and cameras routinely honour the first - // without the others - an integrated webcam offering MJPEG but not at - // 640x480 15fps is what prompted this. So each candidate is tried with the - // constraints loosened a step at a time, giving up the frame rate before - // the size and both before giving up the format, since the format is worth - // the most: it decides whether anything has to be re-encoded at all. - // - // Whatever comes back is what gets announced to the peer - see - // captureStarted() and AudioVideoCallController::video_capture_started() - - // so relaxing these changes what is sent, not what is claimed. - static const struct - { - bool size; - bool rate; - } kConstraints[] = - { - { true, true }, - { true, false }, - { false, false }, - }; + // Which formats to try, in which order, and what to do with each - worked + // out from what the camera said it can deliver at this size and rate. See + // buildCaptureAttempts(). + const QList<CaptureAttempt> attempts = + buildCaptureAttempts(m_deviceName, m_width, m_height, m_frameRate, + m_preferCompressed, m_codec); QString passthroughCodec; AVFormatContext *ifmt_ctx = nullptr; int ret = AVERROR(EINVAL); - bool opened = false; - for (int a = 0; a < attempts.size() && !opened; a++) + for (int a = 0; a < attempts.size(); a++) { - for (unsigned int c = 0; c < sizeof(kConstraints) / sizeof(kConstraints[0]); c++) - { - ret = openCamera(ifmt, deviceUtf8, - kConstraints[c].size ? m_width : 0, - kConstraints[c].size ? m_height : 0, - kConstraints[c].rate ? m_frameRate : 0, - attempts.at(a), &ifmt_ctx); + const CaptureAttempt &attempt = attempts.at(a); - if (ret >= 0) - { - passthroughCodec = attempts.at(a); - opened = true; - break; - } + ret = openCamera(ifmt, deviceUtf8, + attempt.constrainSize ? m_width : 0, + attempt.constrainSize ? m_height : 0, + attempt.constrainRate ? m_frameRate : 0, + attempt.inputFormat, &ifmt_ctx); - QString asked; + if (ret >= 0) + { + // Only a forwarding attempt makes this a passthrough; the same + // format asked for to get the frame rate goes to the transcode + // loop instead. + passthroughCodec = attempt.passthrough ? attempt.inputFormat : QString(); + break; + } - if (!kConstraints[c].size) - asked = QStringLiteral("any size or rate"); - else if (kConstraints[c].rate) - asked = QString("%1x%2 @%3").arg(m_width).arg(m_height).arg(m_frameRate); - else - asked = QString("%1x%2 at any rate").arg(m_width).arg(m_height); + QString asked; - qWarning() << QString("Camera '%1': no %2 at %3 (%4)") - .arg(m_deviceName, - attempts.at(a).isEmpty() ? QStringLiteral("stream") : attempts.at(a), - asked, - ff_errstr(ret)); - } + if (!attempt.constrainSize) + asked = QStringLiteral("any size or rate"); + else if (attempt.constrainRate) + asked = QString("%1x%2 @%3").arg(m_width).arg(m_height).arg(m_frameRate); + else + asked = QString("%1x%2 at any rate").arg(m_width).arg(m_height); + + qWarning() << QString("Camera '%1': no %2 at %3 - %4 (%5)") + .arg(m_deviceName, + attempt.inputFormat.isEmpty() ? QStringLiteral("stream") : attempt.inputFormat, + asked, + attempt.reason, + ff_errstr(ret)); } if (ret < 0) @@ -957,6 +1189,25 @@ void VideoDevice::emitFragments(const unsigned char *data, int size) } } +// The frame rate the camera actually settled on, which is not necessarily the +// one that was asked for - and when the open had to fall back to letting the +// camera choose, was not asked for at all. Encoding at a rate the frames do +// not arrive at gives a stream whose timestamps disagree with reality, and it +// is also what captureStarted() reports, so that the rate a camera quietly +// substituted is visible rather than having to be measured. +static AVRational cameraFrameRate(AVStream *in_st, int fallbackRate) +{ + AVRational fr = in_st->avg_frame_rate; + + if (fr.num <= 0 || fr.den <= 0) + fr = in_st->r_frame_rate; + + if (fr.num <= 0 || fr.den <= 0) + fr = AVRational{ fallbackRate, 1 }; + + return fr; +} + void VideoDevice::runPassthroughLoop(AVFormatContext *ifmt_ctx, AVStream *in_st, const QString &codec) { // The camera decides the picture size here, since nothing rescales it on @@ -964,7 +1215,8 @@ void VideoDevice::runPassthroughLoop(AVFormatContext *ifmt_ctx, AVStream *in_st, int width = in_st->codecpar->width > 0 ? in_st->codecpar->width : m_width; int height = in_st->codecpar->height > 0 ? in_st->codecpar->height : m_height; - emit captureStarted(codec, width, height); + emit captureStarted(codec, width, height, + av_q2d(cameraFrameRate(in_st, m_frameRate))); // Nothing encodes here at all - the camera's own packets go out as they // arrive - so any hardware encoding setting simply has nothing to act on. @@ -995,21 +1247,38 @@ void VideoDevice::runPassthroughLoop(AVFormatContext *ifmt_ctx, AVStream *in_st, av_packet_free(&pkt); } -// The frame rate the camera actually settled on, which is not necessarily the -// one that was asked for - and when the open had to fall back to letting the -// camera choose, was not asked for at all. Encoding at a rate the frames do -// not arrive at gives a stream whose timestamps disagree with reality. -static AVRational cameraFrameRate(AVStream *in_st, int fallbackRate) +// The pixel format to hand libswscale for a decoded camera frame, and whether +// that frame's values use the full 0-255 range rather than the 16-235 one. +// +// MJPEG decodes to one of the yuvj* formats, which mean "the yuv* of the same +// name, with full range values". libswscale still understands them and infers +// the range, but warns that they are deprecated - and since capturing MJPEG +// and re-encoding it is now the ordinary path for anything above VGA on USB +// 2.0, that warning would otherwise be printed for most calls. Saying the same +// thing the way libswscale asks for it keeps the range explicit instead of +// inferred, and the log readable. +static AVPixelFormat swscaleSourceFormat(AVPixelFormat format, bool *fullRange) { - AVRational fr = in_st->avg_frame_rate; + *fullRange = true; - if (fr.num <= 0 || fr.den <= 0) - fr = in_st->r_frame_rate; - - if (fr.num <= 0 || fr.den <= 0) - fr = AVRational{ fallbackRate, 1 }; + switch (format) + { + case AV_PIX_FMT_YUVJ420P: + return AV_PIX_FMT_YUV420P; + case AV_PIX_FMT_YUVJ422P: + return AV_PIX_FMT_YUV422P; + case AV_PIX_FMT_YUVJ444P: + return AV_PIX_FMT_YUV444P; + case AV_PIX_FMT_YUVJ440P: + return AV_PIX_FMT_YUV440P; + case AV_PIX_FMT_YUVJ411P: + return AV_PIX_FMT_YUV411P; + default: + break; + } - return fr; + *fullRange = false; + return format; } void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) @@ -1173,9 +1442,19 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) } } + // Which format the camera is actually delivering, reported alongside the + // encoder because the two together explain the frame rate: an uncompressed + // camera stream is what caps the rate on a USB 2.0 bus, and seeing + // "mjpeg" here is what says that cap has been stepped around. See + // buildCaptureAttempts(). + const QString source = in_st->codecpar->codec_id == AV_CODEC_ID_RAWVIDEO + ? QStringLiteral("uncompressed") + : QString::fromLatin1(avcodec_get_name(in_st->codecpar->codec_id)); + emit accelerationInUse(hardware - ? QString("GPU encoding (VAAPI %1)").arg(QLatin1String(kVideoEncoders[encIdx].codec)) - : QString("CPU encoding (%1)").arg(swEncoderName)); + ? QString("GPU encoding (VAAPI %1) from %2") + .arg(QLatin1String(kVideoEncoders[encIdx].codec), source) + : QString("CPU encoding (%1) from %2").arg(swEncoderName, source)); // Everything the camera sends is scaled to the encoder's size and re-encoded, // so this is what the peer will receive regardless of what the camera @@ -1183,8 +1462,12 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) // out literally, so that changing the encoder above cannot leave the peer // being told something else - avcodec_get_name() yields exactly the // FFmpeg codec names the 0x75 message is defined in terms of. + // + // The rate is the camera's, not the encoder's: the encoder is configured + // from it (see frameRate above) and encodes every frame it is given, so + // what the camera delivers is what goes out. emit captureStarted(QLatin1String(avcodec_get_name(enc_ctx->codec_id)), - enc_ctx->width, enc_ctx->height); + enc_ctx->width, enc_ctx->height, av_q2d(frameRate)); // What the camera's frames are scaled into. With a hardware encoder this // is only a staging buffer in system memory - NV12, which is what VAAPI @@ -1247,7 +1530,11 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) // what was requested. if (!sws) { - sws = sws_getContext(cam_frame->width, cam_frame->height, (AVPixelFormat)cam_frame->format, + bool srcFullRange = false; + const AVPixelFormat srcFormat = + swscaleSourceFormat((AVPixelFormat)cam_frame->format, &srcFullRange); + + sws = sws_getContext(cam_frame->width, cam_frame->height, srcFormat, enc_ctx->width, enc_ctx->height, sw_pix_fmt, SWS_BILINEAR, nullptr, nullptr, nullptr); if (!sws) @@ -1260,6 +1547,18 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) av_frame_unref(cam_frame); continue; } + + // The range the format name no longer carries. The encoder + // wants the studio range, so this is a real conversion and not + // just a label: without it a JPEG frame's blacks and whites + // would be clipped rather than compressed into 16-235. + if (srcFullRange) + { + const int *coefs = sws_getCoefficients(SWS_CS_DEFAULT); + + sws_setColorspaceDetails(sws, coefs, 1, coefs, 0, + 0, 1 << 16, 1 << 16); + } } if (av_frame_make_writable(enc_frame) < 0) diff --git a/pica-client/videodevice.h b/pica-client/videodevice.h index f65895d..02be737 100644 --- a/pica-client/videodevice.h +++ b/pica-client/videodevice.h @@ -64,6 +64,31 @@ private: bool m_haveLastSeq; }; +// One pixel format a camera can deliver a particular size in, with the frame +// rates it can deliver that combination at. +// +// The rates are kept per format rather than merged across them because they +// genuinely differ, by a lot. Uncompressed video at 30fps needs more bandwidth +// than USB 2.0 has for anything much above VGA, so a camera that manages +// 1280x720 at 30fps in MJPEG manages only 10fps uncompressed, and 1920x1080 at +// 5fps. A camera asked for more than a format can carry does not refuse: it +// reports success and quietly sends fewer frames. Keeping this association is +// what lets VideoDevice::Capture() ask for a format that can actually deliver +// what was requested instead of finding out afterwards. +struct VideoPixelFormat +{ + // FFmpeg codec name - "mjpeg", "h264", "hevc" - spelled as the v4l2 + // demuxer's input_format option and the 0x75 message spell it. Empty for + // an uncompressed format, which is what a camera gives when asked for + // nothing in particular. + QString codec; + + // Frame rates available at this size in this format, in whole frames per + // second, highest first. Empty when the camera would not say - which is + // not the same as "none", see VideoDevice::Capture(). + QList<int> frameRates; +}; + // One picture size a camera can deliver, and what it can deliver it in. // Reported by VideoDevice::CaptureFormats() and shown by the settings dialog's // resolution and frame rate lists. @@ -72,10 +97,17 @@ struct VideoCaptureFormat int width; int height; - // Frame rates the camera offers at this size, in whole frames per second, - // highest first. Empty when the camera would not say - which is not the - // same as "none": the capture code can still open it and let the camera - // pick, see VideoDevice::Capture(). + // Every pixel format this size is available in, uncompressed first and + // then the compressed ones in the same order as compressedFormats below. + // This is the detail the two lists after it flatten away. + QList<VideoPixelFormat> pixelFormats; + + // Frame rates the camera offers at this size in its best format for the + // purpose - the union of the rates in pixelFormats. What the size can + // manage at all, rather than what any one format can: reaching the top of + // this list may mean capturing compressed and re-encoding, which is + // Capture()'s business and not the caller's. Highest first, and empty when + // the camera would not say. QList<int> frameRates; // Compressed formats available at this size, as FFmpeg codec names, most @@ -87,6 +119,22 @@ struct VideoCaptureFormat QStringList compressedFormats; }; +// Records that a size is available in `codec` - empty for uncompressed - at +// `rates`, adding to that format's entry rather than replacing it, since two +// of a platform's pixel formats can map to the same codec name. The rates go +// into the size's own union as well. +// +// Shared by the v4l2 and the DirectShow enumerator so that both build the same +// shape out of quite different source data. +void pica_merge_capture_rates(VideoCaptureFormat &entry, const QString &codec, + const QList<int> &rates); + +// Puts one size's formats into the order VideoCaptureFormat documents and sorts +// every rate list highest first. Call once per size, after all of its formats +// have been merged in and compressedFormats has been put in the platform's +// preference order - that order is what pixelFormats is then arranged to match. +void pica_finish_capture_format(VideoCaptureFormat &entry); + // Video counterpart of AudioDevice: a single instance is dedicated to one // direction - either capture+encode (Capture(), reading from a local camera) // or decode (Play(), fed from the network via enqueueFrame()). Each instance @@ -200,7 +248,15 @@ signals: // being sent is known - either the camera's own compressed format, when // forwarding it untouched, or the one this class encodes to. Carries the // values to announce to the peer, before any packet is emitted. - void captureStarted(QString codec, int width, int height); + // + // frameRate is what the camera settled on, which is not necessarily what + // configureCapture() asked for: a camera asked for a rate a pixel format + // cannot carry opens anyway and then delivers fewer frames. Fractional + // because cameras really do offer rates like 7.5fps. Falls back to the + // requested rate when the camera would not say what it is doing, so a + // shortfall it kept quiet about shows up in the measured rate instead - + // see the settings dialog's fps counter. + void captureStarted(QString codec, int width, int height, double frameRate); // One fragment of an encoded frame, ready to be sent over the network. // isLastFragment marks the final fragment of a frame - it maps directly diff --git a/pica-client/videodshow.cpp b/pica-client/videodshow.cpp index 9b389e9..03edbd6 100644 --- a/pica-client/videodshow.cpp +++ b/pica-client/videodshow.cpp @@ -448,12 +448,16 @@ QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device) const QString fourcc = fourccToString(bih->biCompression); + // Empty for an uncompressed media type, which is what + // pica_merge_capture_rates() wants for one. + QString codec; + for (unsigned int f = 0; f < sizeof(kDshowCompressedFormats) / sizeof(kDshowCompressedFormats[0]); f++) { if (fourcc != QLatin1String(kDshowCompressedFormats[f].fourcc)) continue; - QString codec = QLatin1String(kDshowCompressedFormats[f].ffmpeg_codec); + codec = QLatin1String(kDshowCompressedFormats[f].ffmpeg_codec); if (!entry.compressedFormats.contains(codec)) entry.compressedFormats << codec; @@ -468,10 +472,17 @@ QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device) const double maxFps = intervalToFps(scc->MinFrameInterval); const double minFps = intervalToFps(scc->MaxFrameInterval); + // This media type is one size in one format, so the + // rates it offers belong to that format - an + // uncompressed pin capping out well below a compressed + // one at the same size is the whole reason they are + // kept apart, see VideoPixelFormat. + QList<int> rates; + int nominal = qRound(intervalToFps(avgTimePerFrame)); - if (nominal > 0 && !entry.frameRates.contains(nominal)) - entry.frameRates << nominal; + if (nominal > 0) + rates << nominal; // A pin that accepts a range of intervals will take // anything within it, so there is no list to show - @@ -481,10 +492,12 @@ QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device) { const int fps = kDshowStandardFrameRates[r]; - if (fps >= minFps && fps <= maxFps && !entry.frameRates.contains(fps)) - entry.frameRates << fps; + if (fps >= minFps && fps <= maxFps && !rates.contains(fps)) + rates << fps; } + pica_merge_capture_rates(entry, codec, rates); + freeMediaType(mt); } } @@ -519,7 +532,7 @@ QList<VideoCaptureFormat> pica_dshow_capture_formats(const QString &device) entry.compressedFormats = ordered; - std::sort(entry.frameRates.begin(), entry.frameRates.end(), std::greater<int>()); + pica_finish_capture_format(entry); result << entry; } |