From 136bf2eebca5c4e34cdabf23806007d4a7e141c3 Mon Sep 17 00:00:00 2001 From: chenhao Date: Thu, 2 Jan 2025 09:37:42 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=B8=8D=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=E8=BF=9C=E7=AB=AF=E8=A7=86=E9=A2=91=E6=B5=81=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=8C=E4=BC=98=E5=8C=96=E6=97=B6=E9=97=B4=E6=88=B3?= =?UTF-8?q?=E7=9A=84=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: chenhao --- .../ohos_webrtc/camera/camera_capturer.cpp | 2 + .../ohos_webrtc/video/video_frame_receiver.h | 48 +++++++++++++++++++ .../video/video_frame_receiver_gl.cpp | 6 ++- .../ohos_webrtc/video/video_track_source.cpp | 4 +- .../ohos_webrtc/video/video_track_source.h | 2 + .../video_codec/hardware_video_decoder.cpp | 3 +- 6 files changed, 60 insertions(+), 5 deletions(-) diff --git a/sdk/ohos/src/ohos_webrtc/camera/camera_capturer.cpp b/sdk/ohos/src/ohos_webrtc/camera/camera_capturer.cpp index 8ab97e3518..484fe8aa5f 100644 --- a/sdk/ohos/src/ohos_webrtc/camera/camera_capturer.cpp +++ b/sdk/ohos/src/ohos_webrtc/camera/camera_capturer.cpp @@ -69,6 +69,8 @@ void CameraCapturer::Init(std::unique_ptr dataReceiver, Vide dataReceiver_ = std::move(dataReceiver); dataReceiver_->SetVideoFrameSize(profile_.resolution.width, profile_.resolution.height); dataReceiver_->SetCallback(this); + dataReceiver_->SetTimestampConverter( + TimestampConverter([](int64_t timestamp) { return TimestampCast(timestamp); })); isInitialized_ = true; } diff --git a/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver.h b/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver.h index c876635b93..beca36ffd8 100644 --- a/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver.h +++ b/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver.h @@ -24,6 +24,46 @@ namespace webrtc { +template +constexpr Rep TimestampCast(Rep timestamp) +{ + using CF = std::ratio_divide; + + constexpr bool numIsOne = CF::num == 1; + constexpr bool denIsOne = CF::den == 1; + + if constexpr (denIsOne) { + if constexpr (numIsOne) { + return timestamp; + } else { + return timestamp * CF::num; + } + } else { + if constexpr (numIsOne) { + return timestamp / CF::den; + } else { + return timestamp * CF::num / CF::den; + } + } +} + +class TimestampConverter { +public: + TimestampConverter() {} + explicit TimestampConverter(std::function converter) : converter_(std::move(converter)) {} + + int64_t Convert(int64_t timestamp) + { + if (converter_) { + return converter_(timestamp); + } + return timestamp; + } + +private: + std::function converter_; +}; + class VideoFrameReceiver { public: class Callback { @@ -42,8 +82,16 @@ public: callback_ = observer; } + void SetTimestampConverter(TimestampConverter timestampConverter) + { + timestampConverter_ = std::move(timestampConverter); + } + protected: Callback* callback_{}; + // Adapt with different timestamp units from different sources, such as camera and video decoder, + // do nothing by Default. + TimestampConverter timestampConverter_; }; } // namespace webrtc diff --git a/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver_gl.cpp b/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver_gl.cpp index 163b67bc86..70c6e4c2e3 100644 --- a/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver_gl.cpp +++ b/sdk/ohos/src/ohos_webrtc/video/video_frame_receiver_gl.cpp @@ -166,12 +166,14 @@ void VideoFrameReceiverGl::OnNativeImageFrameAvailable() auto matrix = nativeImage_.GetTransformMatrix(); RTC_DLOG(LS_VERBOSE) << "timestamp: " << timestamp; + auto timestampUs = timestampConverter_.Convert(timestamp); + RTC_DLOG(LS_VERBOSE) << "timestampUs=" << timestampUs; + // create video frame auto buffer = TextureBuffer::Create(textureData_, width_, height_, matrix); if (callback_) { - callback_->OnFrameAvailable( - buffer, timestamp > 0 ? (timestamp / rtc::kNumNanosecsPerMicrosec) : rtc::TimeMicros(), kVideoRotation_0); + callback_->OnFrameAvailable(buffer, timestampUs > 0 ? timestampUs : rtc::TimeMicros(), kVideoRotation_0); } } diff --git a/sdk/ohos/src/ohos_webrtc/video/video_track_source.cpp b/sdk/ohos/src/ohos_webrtc/video/video_track_source.cpp index b085bbfb12..8849b62024 100644 --- a/sdk/ohos/src/ohos_webrtc/video/video_track_source.cpp +++ b/sdk/ohos/src/ohos_webrtc/video/video_track_source.cpp @@ -253,6 +253,8 @@ void OhosVideoTrackSource::OnFrameCaptured( rtc::scoped_refptr buffer, int64_t timestampUs, VideoRotation rotation) { RTC_DLOG(LS_VERBOSE) << __FUNCTION__ << " timestampUs=" << timestampUs << ", rotation=" << rotation; + auto alignedTimestampUs = timestampAligner_.TranslateTimestamp(timestampUs, rtc::TimeMicros()); + RTC_DLOG(LS_VERBOSE) << "alignedTimestampUs=" << alignedTimestampUs; int adapted_width = 0; int adapted_height = 0; @@ -292,7 +294,7 @@ void OhosVideoTrackSource::OnFrameCaptured( auto frame = VideoFrame::Builder() .set_video_frame_buffer(buffer) .set_rotation(rotation) - .set_timestamp_us(timestampUs) + .set_timestamp_us(alignedTimestampUs) .build(); if (ApplyRotation() && frame.rotation() != kVideoRotation_0 && buffer->type() == VideoFrameBuffer::Type::kI420) { diff --git a/sdk/ohos/src/ohos_webrtc/video/video_track_source.h b/sdk/ohos/src/ohos_webrtc/video/video_track_source.h index 7af33dab49..aa13f623fb 100644 --- a/sdk/ohos/src/ohos_webrtc/video/video_track_source.h +++ b/sdk/ohos/src/ohos_webrtc/video/video_track_source.h @@ -25,6 +25,7 @@ #include "media/base/video_broadcaster.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread.h" +#include "rtc_base/timestamp_aligner.h" #include @@ -99,6 +100,7 @@ private: std::optional stats_ RTC_GUARDED_BY(statsMutex_); std::mutex obsMutex_; VideoCapturer::Observer* capturerObserver_{}; + rtc::TimestampAligner timestampAligner_; }; } // namespace webrtc diff --git a/sdk/ohos/src/ohos_webrtc/video_codec/hardware_video_decoder.cpp b/sdk/ohos/src/ohos_webrtc/video_codec/hardware_video_decoder.cpp index 21661fcab2..750746dad0 100644 --- a/sdk/ohos/src/ohos_webrtc/video_codec/hardware_video_decoder.cpp +++ b/sdk/ohos/src/ohos_webrtc/video_codec/hardware_video_decoder.cpp @@ -401,8 +401,7 @@ void HardwareVideoDecoder::DeliverTextureFrame(const ohos::CodecBuffer& buffer) << ", pts=" << attr.pts; RTC_DLOG(LS_VERBOSE) << "render output buffer, index=" << buffer.index; - ret = - OH_VideoDecoder_RenderOutputBufferAtTime(decoder_.Raw(), buffer.index, attr.pts * rtc::kNumNanosecsPerMicrosec); + ret = OH_VideoDecoder_RenderOutputBuffer(decoder_.Raw(), buffer.index); if (ret != AV_ERR_OK) { RTC_LOG(LS_ERROR) << "Failed to render output buffer" << ret; } -- Gitee