Fixes for correctly setting encoder frame rate for various capture modes

Author: GennadiyAMD

README.md

@@ -4,6 +4,8 @@ Advanced Interactive Streaming SDK is a C++ library that provides building block
 Virtual and Augmented Reality, etc. using AMD Radeon graphics. It allows you to build a complete streaming solution including video and audio capture, video and audio encoder/decoder/pre-post-processing pipelines, a robust and secure network stack, or use some of its components, while implementing the rest yourself.
 
 ## Changelog:
+- v1.0.2 - bug fixes:
+	- Encoder frame rate wasn't being set correctly when display capture mode was set to *present* or when QoS was turned off
 - v1.0.1 - bug fixes:
 	- System mouse cursor doesn't get hidden correctly in some games
 	- Deadlock on server exit when streaming at high (8K) resolution

samples/RemoteDesktopServer/AVStreamer.cpp

@@ -464,16 +464,6 @@ void AVStreamer::OnFramerateChangeByQoS(float framerate, ssdk::transport_common:
         {
             AMFTraceError(AMF_FACILITY, L"QoS failed to set video capture frame rate for stream %lld to %5.2f, result=%s", streamID, framerate, amf::AMFGetResultText(result));
         }
-
-        result = videoOutput->SetFramerate(framerate);
-        if (result == AMF_OK)
-        {
-            AMFTraceDebug(AMF_FACILITY, L"Encoder frame rate for stream %lld has been changed to %5.2f by QoS", streamID, framerate);
-        }
-        else
-        {
-            AMFTraceError(AMF_FACILITY, L"QoS failed to set encoder frame rate for stream %lld to %5.2f, result=%s", streamID, framerate, amf::AMFGetResultText(result));
-        }
     }
 }
 

samples/RemoteDesktopServer/RemoteDesktopServer.cpp

@@ -482,7 +482,7 @@ bool RemoteDesktopServer::InitVideoCapture()
                 break;
             case AMF_NOT_FOUND:
             case AMF_ACCESS_DENIED:
-                AMFTraceWarning(AMF_FACILITY, L"Selected video capture method does not support capture at a fixed frame rate, -%s %S ignored", PARAM_NAME_CAPTURE_MODE, CAPTURE_MODE_FRAMERATE);
+                AMFTraceWarning(AMF_FACILITY, L"Selected video capture method does not support capture at a fixed frame rate, -%s %s ignored", PARAM_NAME_CAPTURE_MODE, CAPTURE_MODE_FRAMERATE);
                 result = true;
                 break;
             default:
@@ -493,20 +493,22 @@ bool RemoteDesktopServer::InitVideoCapture()
         else if (captureMode == CAPTURE_MODE_PRESENT)
         {
             amfResult = m_VideoCapture->SetProperty(AMF_DISPLAYCAPTURE_MODE, amf_int64(AMF_DISPLAYCAPTURE_MODE_ENUM::AMF_DISPLAYCAPTURE_MODE_WAIT_FOR_PRESENT));
+            m_VideoStreamDescriptor.SetFramerate(60.0); //  Set the initial frame rate to 60fps, it will be adjusted automatically in a few seconds to the actual frame rate determined by capture
         }
         else if (captureMode == CAPTURE_MODE_ASAP)
         {
             amfResult = m_VideoCapture->SetProperty(AMF_DISPLAYCAPTURE_MODE, amf_int64(AMF_DISPLAYCAPTURE_MODE_ENUM::AMF_DISPLAYCAPTURE_MODE_GET_CURRENT_SURFACE));
+            m_VideoStreamDescriptor.SetFramerate(60.0); //  Set the initial frame rate to 60fps, it will be adjusted automatically in a few seconds to the actual frame rate determined by capture
         }
         if (result != true)
         {
             if (amfResult == AMF_OK)
             {
-                AMFTraceInfo(AMF_FACILITY, L"Capture mode is set to %S", captureMode.c_str());
+                AMFTraceInfo(AMF_FACILITY, L"Capture mode is set to %s", captureMode.c_str());
             }
             else
             {
-                AMFTraceError(AMF_FACILITY, L"Failed to set video capture mode to %S, result=%s", captureMode.c_str(), amf::AMFGetResultText(amfResult));
+                AMFTraceError(AMF_FACILITY, L"Failed to set video capture mode to %s, result=%s", captureMode.c_str(), amf::AMFGetResultText(amfResult));
             }
         }
 

sdk/util/stats/ClientStatsManager.cpp

@@ -59,7 +59,7 @@ namespace ssdk::util
 
     void ClientStatsManager::Clear()
     {
-        m_FramerateHistory.Clear();
+        m_FrameDurationHistory.Clear();
         m_FullLatencyHistory.Clear();
         m_ClientLatencyHistory.Clear();
         m_ServerLatencyHistory.Clear();
@@ -97,9 +97,9 @@ namespace ssdk::util
     void ClientStatsManager::UpdateFullLatencyAndFramerate(amf_pts fullLatencyPts)
     {
         amf_pts now = amf_high_precision_clock();
-        float framerate = (float)AMF_SECOND / (now - m_LastFrameTime);
+        float frameDuration = (float)(now - m_LastFrameTime) / AMF_SECOND;
         m_LastFrameTime = now;
-        m_FramerateHistory.Add(framerate);
+        m_FrameDurationHistory.Add(frameDuration);
         float fullLatency = (float)(fullLatencyPts) / AMF_MILLISECOND;
         m_FullLatencyHistory.Add(fullLatency);
     }
@@ -155,7 +155,12 @@ namespace ssdk::util
                 float decrypt = m_DecryptHistory.GetAverageAndClear();
                 int32_t decoderQueueSize = static_cast<int32_t>(m_DecoderQueueTimes.size());
                 float AVDesync = m_AVDesyncHistory.GetAverageAndClear();
-                float framerate = m_FramerateHistory.GetAverageAndClear();
+                float frameDurationAvg = m_FrameDurationHistory.GetAverageAndClear();
+                float framerate = 0;
+                if (frameDurationAvg != 0)
+                {
+                    framerate = 1 / frameDurationAvg;
+                }
 
                 StatsSnapshotImpl statsSnapshotImpl(fullLatency, clientLatency, serverLatency, encoderLatency, networkLatency, decoderLatency, decrypt, decoderQueueSize, AVDesync, framerate);
                 transport->SendStats(ssdk::transport_common::DEFAULT_STREAM, statsSnapshotImpl);

sdk/util/stats/ClientStatsManager.h

@@ -111,7 +111,7 @@ namespace ssdk::util
         ssdk::transport_common::ClientTransport* m_Transport = nullptr;
 
         typedef ssdk::util::ValueAverage<float> FloatValueAverage;
-        FloatValueAverage m_FramerateHistory;
+        FloatValueAverage m_FrameDurationHistory;
         FloatValueAverage m_FullLatencyHistory;
         FloatValueAverage m_ClientLatencyHistory;
         FloatValueAverage m_ServerLatencyHistory;

sdk/video/MonoscopicVideoOutput.cpp

@@ -222,7 +222,12 @@ namespace ssdk::video
             amf::AMFLock lock(&m_Guard);
             m_Bitrate = bitrate;
         }
-        return m_Encoder->UpdateBitrate(bitrate);
+        AMF_RESULT result = m_Encoder->UpdateBitrate(bitrate);
+        if (result != AMF_OK)
+        {
+            AMFTraceError(AMF_FACILITY, L"Failed to set video encoder bitrate rate to %lld", bitrate);
+        }
+        return result;
     }
 
     float MonoscopicVideoOutput::GetFramerate() const noexcept
@@ -237,7 +242,12 @@ namespace ssdk::video
             amf::AMFLock lock(&m_Guard);
             m_FrameRate = framerate;
         }
-        return m_Encoder->UpdateFramerate(AMFConstructRate(uint32_t(framerate), 1));
+        AMF_RESULT result = m_Encoder->UpdateFramerate(AMFConstructRate(uint32_t(framerate), 1));
+        if (result != AMF_OK)
+        {
+            AMFTraceError(AMF_FACILITY, L"Failed to set video encoder frame rate to %5.2f", framerate);
+        }
+        return result;
     }
 
     bool MonoscopicVideoOutput::IsKeyFrameRequested() noexcept
@@ -302,6 +312,26 @@ namespace ssdk::video
             }
             converter = m_Converter;    //  Also saving a pointer to the converter in a local variable to avoid unnecessary locking, only using locals below.
                                         //  The encoder is always present because it's passed to a constructor, so no need to worry about it changing on the fly
+
+            //  Adjust the frame rate setting on the encoder to reflect the actual measured frame rate when frames are captured as Present or ASAP
+            amf_pts now = amf_high_precision_clock();
+            if (m_FrameCnt == 0)
+            {
+                m_FrameCntStartTime = now;
+            }
+            ++m_FrameCnt;
+            constexpr static amf_pts FPS_MEASUREMENT_PERIOD = 3;
+            amf_pts timeSinceHistoryStart = now - m_FrameCntStartTime;
+            if (timeSinceHistoryStart > FPS_MEASUREMENT_PERIOD * AMF_SECOND)
+            {
+                AMFRate fps = { uint32_t(m_FrameCnt / (timeSinceHistoryStart / AMF_SECOND)), 1 };
+                float prevFps = m_Encoder->GetFramerate();
+                if (abs(fps.num - prevFps) > prevFps * 0.1) //  We don't want to change the encoder's frame rate too frequently by a small amount because it can reduce image quality, but when the frame changes significantly, we want to reflect that for a more optimal compression
+                {
+                    m_Encoder->UpdateFramerate(fps);
+                }
+                m_FrameCnt = 0;
+            }
         }
 
         input->SetProperty(ORIGIN_PTS_PROPERTY, originPts);

sdk/video/MonoscopicVideoOutput.h

@@ -146,5 +146,8 @@ namespace ssdk::video
 
         int64_t                     m_SequenceNumber = 0;
         int64_t                     m_FramesSubmitted = 0;
+
+        amf_pts                     m_FrameCntStartTime = 0;
+        uint32_t                    m_FrameCnt = 0;
     };
 }

sdk/video/encoders/GPUEncoderAV1.cpp

@@ -155,7 +155,12 @@ namespace ssdk::video
     AMF_RESULT GPUEncoderAV1::UpdateFramerate(const AMFRate& rate)
     {
         AMF_RETURN_IF_FALSE(m_Encoder != nullptr, AMF_FAIL, L"Encoder is not initialized");
-        return m_Encoder->SetProperty(AMF_VIDEO_ENCODER_AV1_FRAMERATE, rate);
+        AMF_RESULT result = m_Encoder->SetProperty(AMF_VIDEO_ENCODER_AV1_FRAMERATE, rate);
+        if (result == AMF_OK)
+        {
+            m_Framerate = float(rate.num) / float(rate.den);
+        }
+        return result;
     }
 
     AMF_RESULT GPUEncoderAV1::GetExtraData(amf::AMFBuffer** pExtraData) const

sdk/video/encoders/GPUEncoderH264.cpp

@@ -134,7 +134,12 @@ namespace ssdk::video
     AMF_RESULT GPUEncoderH264::UpdateFramerate(const AMFRate& rate)
     {
         AMF_RETURN_IF_FALSE(m_Encoder != nullptr, AMF_FAIL, L"Encoder is not initialized");
-        return m_Encoder->SetProperty(AMF_VIDEO_ENCODER_FRAMERATE, rate);
+        AMF_RESULT result = m_Encoder->SetProperty(AMF_VIDEO_ENCODER_FRAMERATE, rate);
+        if (result == AMF_OK)
+        {
+            m_Framerate = float(rate.num) / float(rate.den);
+        }
+        return result;
     }
 
     AMF_RESULT GPUEncoderH264::GetExtraData(amf::AMFBuffer** pExtraData) const

sdk/video/encoders/GPUEncoderHEVC.cpp

@@ -163,7 +163,12 @@ namespace ssdk::video
     AMF_RESULT GPUEncoderHEVC::UpdateFramerate(const AMFRate& rate)
     {
         AMF_RETURN_IF_FALSE(m_Encoder != nullptr, AMF_FAIL, L"Encoder is not initialized");
-        return m_Encoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_FRAMERATE, rate);
+        AMF_RESULT result =  m_Encoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_FRAMERATE, rate);
+        if (result == AMF_OK)
+        {
+            m_Framerate = float(rate.num) / float(rate.den);
+        }
+        return result;
     }
 
     AMF_RESULT GPUEncoderHEVC::GetExtraData(amf::AMFBuffer** pExtraData) const