diff --git a/app/src/main/res/layout/cas_activity_fullscreen.xml b/app/src/main/res/layout/cas_activity_fullscreen.xml index dd685cb17503954d1927851601d3f22c34d574f1..441825792ebe867b376c081784543f80df7c2772 100644 --- a/app/src/main/res/layout/cas_activity_fullscreen.xml +++ b/app/src/main/res/layout/cas_activity_fullscreen.xml @@ -38,7 +38,7 @@ GetNow() / 1000; + uint64_t *pCurrentTime = reinterpret_cast(outBuffer + sizeof(stream_msg_head_t) + pkgLen); + *pCurrentTime = hton64(currentTime); +#endif + int ret = m_mtrans->SendTouchEventData(reinterpret_cast(outBuffer), dataLen); if (ret != static_cast(dataLen)) { ERR("Failed to send touch event, aimed to send:%d, sent:%d", (int)dataLen, ret); @@ -1223,6 +1237,22 @@ void CasController::RecvdVideoData(uint8_t *data, int length) if (m_videoPacketStream != nullptr) { uint8_t *videoData = new uint8_t[length]; memcpy(videoData, data, length); +#ifdef ENABLE_E2E_TIME + ((stream_msg_head_t *)videoData)->SetPayloadSize(length - sizeof(VideoFrameOTP)); + + VideoFrameOTP videoFrameOtp {}; + (void)memcpy_s(&videoFrameOtp, sizeof(VideoFrameOTP), videoData + length - sizeof(VideoFrameOTP), sizeof(VideoFrameOTP)); + videoFrameOtp.serverDelay = htonl(videoFrameOtp.serverDelay); + videoFrameOtp.clientSendControlTs = hton64(videoFrameOtp.clientSendControlTs); + videoFrameOtp.clientRecvFrameTs = CasVideoUtil::GetInstance()->GetNow() / 1000; + + uint64_t sendToRecvTime = videoFrameOtp.clientRecvFrameTs - videoFrameOtp.clientSendControlTs; + if (videoFrameOtp.clientSendControlTs > 0) { + CasVideoUtil::GetInstance()->updateVideoFrameOtp(videoFrameOtp); + DBG("CasController::RecvdVideoData serverDelay = %d, touch to recv time = %d, clientSendControlTs = %" PRIu64 "", (int)sendToRecvTime, videoFrameOtp.serverDelay, videoFrameOtp.clientSendControlTs); + } + CasVideoUtil::GetInstance()->RecordVideoFrameBeforeDecode(videoFrameOtp.clientSendControlTs); +#endif m_videoPacketStream->Handle(videoData); CalculateFPS(); } @@ -1375,7 +1405,11 @@ std::string CasController::GetVideoRecvStats() { #endif stream << "帧率 : " << GetCurrentFPS() << "fps" << std::endl; stream << "策略丢帧 : " << CasVideoUtil::GetInstance()->GetCurrentDropFPS() << "fps" << std::endl; - +#ifdef ENABLE_E2E_TIME + stream << "操作出帧时延(需操作) : " << CasVideoUtil::GetInstance()->m_sendToRecvDelay << "ms" << std::endl; + stream << "服务端处理时延(需操作) : " << CasVideoUtil::GetInstance()->m_serverDelay << "ms" << std::endl; + stream << "操作显示时延(需操作) : " << CasVideoUtil::GetInstance()->m_sendToDecodeDelay << "ms" << std::endl; +#endif statsString = stream.str(); return statsString; } diff --git a/cloudphone/src/main/cpp/cas_common/CasMsg.h b/cloudphone/src/main/cpp/cas_common/CasMsg.h index f591ff846bf06edc253bc95cfe1d0f24a3088df5..2f682e1d23128c4ff25c6257c073e0483a199f1c 100644 --- a/cloudphone/src/main/cpp/cas_common/CasMsg.h +++ b/cloudphone/src/main/cpp/cas_common/CasMsg.h @@ -42,6 +42,23 @@ enum CasMsgType : uint8_t { End, }; +static inline bool isBigEndian() { + uint16_t flag = 0xFF; + uint8_t *pFirst = reinterpret_cast(&flag); + return *pFirst == 0; +} + +#define swap64(val) (((val) >> 56) |\ + (((val) & 0x00ff000000000000ll) >> 40) |\ + (((val) & 0x0000ff0000000000ll) >> 24) |\ + (((val) & 0x000000ff00000000ll) >> 8) |\ + (((val) & 0x00000000ff000000ll) << 8) |\ + (((val) & 0x0000000000ff0000ll) << 24) |\ + (((val) & 0x000000000000ff00ll) << 40) |\ + (((val) << 56))) + +#define hton64(val) (isBigEndian() ? val : swap64(val)) +#define ntoh64(val) hton64(val) #define CAS_STREAM_DELIMITER_MAGICWORD 0x5A5A #define GET_CAS_CHECKSUM(msg_type) ((msg_type + ((CAS_STREAM_DELIMITER_MAGICWORD >> 8) & 0xFF) + (CAS_STREAM_DELIMITER_MAGICWORD & 0xFF)) & 0xFF) @@ -108,4 +125,11 @@ typedef struct CasMotionEventMsg { int32_t secondaryValue; } __packed CasMotionEventMsg; +struct VideoFrameOTP { + int serverDelay {0}; + uint64_t clientSendControlTs {0}; + uint64_t clientRecvFrameTs {0}; + uint64_t clientDecodeTs {0}; +}__attribute__((packed)); + #endif // CLOUDAPPSDK_CASMSG_H \ No newline at end of file diff --git a/cloudphone/src/main/cpp/cas_decoder/CasDecoder.cpp b/cloudphone/src/main/cpp/cas_decoder/CasDecoder.cpp index a80cb0101e25d5a9777c7357796e00a074def9f6..19bd039d5ec2aedb95a86c9f0235e26034abf32e 100644 --- a/cloudphone/src/main/cpp/cas_decoder/CasDecoder.cpp +++ b/cloudphone/src/main/cpp/cas_decoder/CasDecoder.cpp @@ -224,6 +224,9 @@ uint32_t CasDecoder::Input(const uint8_t *buf, const size_t len) */ uint32_t CasDecoder::OutputAndDisplay(bool isDisplay) { +#ifdef ENABLE_E2E_TIME + m_casVideoUtil->CalculateVideoFrameOtp(); +#endif AMediaCodecBufferInfo info; // Get the index of the next available buffer of processed data. ssize_t bufId = AMediaCodec_dequeueOutputBuffer(m_mediaCodec, &info, MAX_TIMEOUT_USEC); diff --git a/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.cpp b/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.cpp index f56b7ead3e24673c5b135d8ee1b6d229f82294b0..7f284e9ec7d55f46385c5f7f24bdce06d90fbd2b 100644 --- a/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.cpp +++ b/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "CasVideoUtil.h" #include "CasLog.h" @@ -176,4 +177,38 @@ int CasVideoUtil::GetCurrentDropFPS() { return 0; } return m_VideoDropFPS; -} \ No newline at end of file +} + +void CasVideoUtil::RecordVideoFrameBeforeDecode(uint64_t clientSendTime) { + m_frameSendTimeQueue.push(clientSendTime); +} + +void CasVideoUtil::updateVideoFrameOtp(VideoFrameOTP &videoFrameOtp) { + m_otpRecordMap[videoFrameOtp.clientSendControlTs] = videoFrameOtp; +} + +void CasVideoUtil::CalculateVideoFrameOtp() { + if (m_frameSendTimeQueue.empty()) { + return; + } + uint64_t sendTime = m_frameSendTimeQueue.front(); + m_frameSendTimeQueue.pop(); + + if (sendTime == 0) { + return; + } + if (m_otpRecordMap.count(sendTime) <= 0) { + return; + } + VideoFrameOTP videoFrameOtp = m_otpRecordMap[sendTime]; + videoFrameOtp.clientDecodeTs = GetNow() / 1000; + + uint64_t sendToDecodeTime = videoFrameOtp.clientDecodeTs - videoFrameOtp.clientSendControlTs; + uint64_t sendToRecvTime = videoFrameOtp.clientRecvFrameTs - videoFrameOtp.clientSendControlTs; + + m_serverDelay = videoFrameOtp.serverDelay; + m_sendToRecvDelay = static_cast(sendToRecvTime); + m_sendToDecodeDelay = static_cast(sendToDecodeTime); + + m_otpRecordMap.erase(sendTime); +} diff --git a/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.h b/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.h index f31d48a4c65a71173e16e086f363638e86427321..dad9132aa4111f38808630b74185c49bec6a81a7 100644 --- a/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.h +++ b/cloudphone/src/main/cpp/cas_decoder/CasVideoUtil.h @@ -17,6 +17,9 @@ #include #include +#include +#include +#include "CasMsg.h" #include "CasItemQueue.h" class CasVideoUtil { @@ -72,6 +75,14 @@ public: */ int GetCurrentDropFPS(); + void RecordVideoFrameBeforeDecode(uint64_t clientSendTime); + void updateVideoFrameOtp(VideoFrameOTP &videoFrameOtp); + void CalculateVideoFrameOtp(); + + int m_serverDelay = 0; + int m_sendToRecvDelay = 0; + int m_sendToDecodeDelay = 0; + private: /* * @brief: construct @@ -96,5 +107,8 @@ private: int m_VideoDropCount = 0; int m_VideoDropFPS = 0; uint64_t m_lastTimeRefreshDropFPS = 0; + + std::queue m_frameSendTimeQueue; + std::unordered_map m_otpRecordMap; }; #endif // CLOUDAPPSDK_CASVIDEOUTIL_H \ No newline at end of file