diff --git a/services/cameraservice/base/include/dcamera_frame_info.h b/services/cameraservice/base/include/dcamera_frame_info.h index 042435a462a1ce44016548b4b23886e6a4ac6244..7ec96675a4081b3385ea4ffdde0eaa2c1c3aed10 100644 --- a/services/cameraservice/base/include/dcamera_frame_info.h +++ b/services/cameraservice/base/include/dcamera_frame_info.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Huawei Device Co., Ltd. + * Copyright (c) 2023-2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -38,6 +38,7 @@ struct DCameraFrameInfo { int32_t index = 0; int32_t offset = 0; int64_t pts = 0; + int64_t rawTime = 0; DCameraFrameProcessTimePoint timePonit {0}; }; } // namespace DistributedHardware diff --git a/services/cameraservice/base/include/dcamera_sink_frame_info.h b/services/cameraservice/base/include/dcamera_sink_frame_info.h index d5a8f6153403c929d109e4c00068ca21c37da02c..953a9a886a810081b0ba35e2e98661b4e1b1377b 100644 --- a/services/cameraservice/base/include/dcamera_sink_frame_info.h +++ b/services/cameraservice/base/include/dcamera_sink_frame_info.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Huawei Device Co., Ltd. + * Copyright (c) 2023-2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -33,6 +33,7 @@ public: int64_t finishEncodeT_; int64_t sendT_; std::string ver_; + std::string rawTime_; public: const std::string FRAME_INFO_TYPE = "type"; @@ -43,6 +44,7 @@ public: const std::string FRAME_INFO_ENCODET = "encodeT"; const std::string FRAME_INFO_SENDT = "sendT"; const std::string FRAME_INFO_VERSION = "ver"; + const std::string RAW_TIME = "rawTime"; public: void Marshal(std::string& jsonStr); diff --git a/services/cameraservice/base/src/dcamera_sink_frame_info.cpp b/services/cameraservice/base/src/dcamera_sink_frame_info.cpp index 3290041698d292e25e7cf80093ecf9a8da7e5dec..7cf1a5df0dfb14633245c032ccac4bb1eb8d37bc 100644 --- a/services/cameraservice/base/src/dcamera_sink_frame_info.cpp +++ b/services/cameraservice/base/src/dcamera_sink_frame_info.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Huawei Device Co., Ltd. + * Copyright (c) 2023-2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -33,6 +33,7 @@ void DCameraSinkFrameInfo::Marshal(std::string& jsonStr) cJSON_AddNumberToObject(frameInfo, FRAME_INFO_FINISH_ENCODE.c_str(), finishEncodeT_); cJSON_AddNumberToObject(frameInfo, FRAME_INFO_SENDT.c_str(), sendT_); cJSON_AddStringToObject(frameInfo, FRAME_INFO_VERSION.c_str(), ver_.c_str()); + cJSON_AddStringToObject(frameInfo, RAW_TIME.c_str(), rawTime_.c_str()); char *data = cJSON_Print(frameInfo); if (data == nullptr) { @@ -92,6 +93,13 @@ int32_t DCameraSinkFrameInfo::Unmarshal(const std::string& jsonStr) return DCAMERA_BAD_VALUE; } ver_ = std::string(ver->valuestring); + cJSON *rawTime = cJSON_GetObjectItemCaseSensitive(rootValue, RAW_TIME.c_str()); + if (rawTime == nullptr || !cJSON_IsString(rawTime)) { + DHLOGE("rawTime is null or not string"); + rawTime_ = ""; + } else { + rawTime_ = std::string(rawTime->valuestring); + } cJSON_Delete(rootValue); return DCAMERA_OK; } diff --git a/services/cameraservice/cameraoperator/client/src/dcamera_client.cpp b/services/cameraservice/cameraoperator/client/src/dcamera_client.cpp index 4a300270ec99ca17ddfa37a2fc26ca37178efe97..47cdcc937c49d245c25ab93ed138602d493b69ba 100644 --- a/services/cameraservice/cameraoperator/client/src/dcamera_client.cpp +++ b/services/cameraservice/cameraoperator/client/src/dcamera_client.cpp @@ -433,7 +433,8 @@ int32_t DCameraClient::ConfigCaptureSessionInner() GetAnonyString(cameraId_).c_str(), ret); return ret; } - + ret = ((sptr &)previewOutput_)->SetFrameRate(30, 30); + DHLOGI("SetFrameRate ret: %{public}d", ret); ret = captureSession_->Start(); if (ret != DCAMERA_OK) { DHLOGE("ConfigCaptureSession %{public}s start captureSession failed, ret: %{public}d", diff --git a/services/cameraservice/sourceservice/include/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.h b/services/cameraservice/sourceservice/include/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.h index 6a45d1d3356a96b4d03235fdbd589c8eee446962..11f19037243f4c165200edfc0e2c30bf30c5b146 100644 --- a/services/cameraservice/sourceservice/include/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.h +++ b/services/cameraservice/sourceservice/include/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023 Huawei Device Co., Ltd. + * Copyright (c) 2021-2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -21,11 +21,15 @@ #include #include #include +#include +#include +#include #include "data_buffer.h" #include "event_handler.h" #include "v1_1/id_camera_provider.h" #include "dcamera_feeding_smoother.h" +#include "idistributed_camera_source.h" namespace OHOS { namespace DistributedHardware { @@ -37,6 +41,18 @@ public: DCAMERA_PRODUCER_STATE_STOP = 0, DCAMERA_PRODUCER_STATE_START = 1, } DCameraProducerState; + + struct SyncSharedData { + volatile int lock; + uint64_t audio_current_pts; + uint64_t audio_update_clock; + float audio_speed; + uint64_t video_current_pts; + uint64_t video_update_clock; + float video_speed; + uint64_t sync_strategy; + bool reset; + }; DCameraStreamDataProcessProducer(std::string devId, std::string dhId, int32_t streamId, DCStreamType streamType); ~DCameraStreamDataProcessProducer(); @@ -45,14 +61,27 @@ public: void FeedStream(const std::shared_ptr& buffer); virtual void OnSmoothFinished(const std::shared_ptr& data) override; + void UpdateProducerWorkMode(const WorkModeParam& param); + private: void StartEvent(); void LooperSnapShot(); int32_t FeedStreamToDriver(const DHBase& dhBase, const std::shared_ptr& buffer); int32_t CheckSharedMemory(const DCameraBuffer& sharedMemory, const std::shared_ptr& buffer); + void SyncVideoThread(); + bool WaitForVideoFrame(std::shared_ptr& buffer); + int32_t SyncVideoFrame(uint64_t videoPtsUs); + void UpdateVideoClock(uint64_t videoPtsUs); const uint32_t DCAMERA_PRODUCER_MAX_BUFFER_SIZE = 30; const uint32_t DCAMERA_PRODUCER_RETRY_SLEEP_MS = 500; + const uint32_t DCAMERA_MAX_SYNC_BUFFER_SIZE = 10; + const uint32_t DCAMERA_SYNC_WATERMARK = 1; + const uint32_t DCAMERA_SYNC_TIME_INTERVAL = 33; + const uint32_t DCAMERA_NS_TO_MS = 1000000; + const uint32_t DCAMERA_US_TO_MS = 1000000; + const uint32_t DCAMERA_TIME_DIFF_MAX = 5; + const int32_t DCAMERA_TIME_DIFF_MIN = -80; private: std::string devId_; @@ -75,6 +104,16 @@ private: sptr camHdiProvider_; std::unique_ptr smoother_ = nullptr; std::shared_ptr smootherListener_ = nullptr; + + std::thread syncThread_; + std::atomic syncRunning_; + std::deque> syncBufferQueue_; + std::mutex syncBufferMutex_; + std::condition_variable syncBufferCond_; + std::atomic isFirstFrame_; + std::chrono::steady_clock::time_point startTime_; + WorkModeParam workModeParam_; // Audio-video synchronization fwk transfer structure + sptr syncMem_; // Shared memory }; } // namespace DistributedHardware } // namespace OHOS diff --git a/services/cameraservice/sourceservice/src/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.cpp b/services/cameraservice/sourceservice/src/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.cpp index 97e78bcf3bb40d5c39702f66ec50d5de25307a1d..4cba89c5c9125ac6c325bda1344deb099c1184eb 100644 --- a/services/cameraservice/sourceservice/src/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.cpp +++ b/services/cameraservice/sourceservice/src/distributedcameramgr/dcameradata/dcamera_stream_data_process_producer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023 Huawei Device Co., Ltd. + * Copyright (c) 2021-2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -18,6 +18,7 @@ #include #include +#include "ashmem.h" #include "anonymous_string.h" #include "dcamera_buffer_handle.h" #include "dcamera_hidumper.h" @@ -33,13 +34,15 @@ namespace DistributedHardware { DCameraStreamDataProcessProducer::DCameraStreamDataProcessProducer(std::string devId, std::string dhId, int32_t streamId, DCStreamType streamType) : devId_(devId), dhId_(dhId), streamId_(streamId), streamType_(streamType), eventHandler_(nullptr), - camHdiProvider_(nullptr) + camHdiProvider_(nullptr), workModeParam_(-1, 0, 0, 0) { DHLOGI("DCameraStreamDataProcessProducer Constructor devId %{public}s dhId %{public}s streamType: %{public}d " "streamId: %{public}d", GetAnonyString(devId_).c_str(), GetAnonyString(dhId_).c_str(), streamType_, streamId_); state_ = DCAMERA_PRODUCER_STATE_STOP; interval_ = DCAMERA_PRODUCER_ONE_MINUTE_MS / DCAMERA_PRODUCER_FPS_DEFAULT; photoCount_ = COUNT_INIT_NUM; + syncRunning_.store(false); + isFirstFrame_.store(true); } DCameraStreamDataProcessProducer::~DCameraStreamDataProcessProducer() @@ -71,6 +74,10 @@ void DCameraStreamDataProcessProducer::Start() smootherListener_ = std::make_shared(shared_from_this()); smoother_->RegisterListener(smootherListener_); smoother_->StartSmooth(); + + // Start the audio and video synchronization thread + syncRunning_.store(true); + syncThread_ = std::thread([this]() { this->SyncVideoThread(); }); } else { producerThread_ = std::thread([this]() { this->LooperSnapShot(); }); } @@ -96,6 +103,17 @@ void DCameraStreamDataProcessProducer::Stop() } eventThread_.join(); eventHandler_ = nullptr; + // Stop the audio and video synchronization thread + if (syncMem_ != nullptr) { + syncMem_->UnmapAshmem(); + syncMem_->CloseAshmem(); + syncMem_ = nullptr; + } + syncRunning_.store(false); + syncBufferCond_.notify_all(); + if (syncThread_.joinable()) { + syncThread_.join(); + } } else { producerCon_.notify_one(); producerThread_.join(); @@ -280,6 +298,19 @@ void DCameraStreamDataProcessProducer::OnSmoothFinished(const std::shared_ptrData(), buffer->Size()); } #endif + // Check if audio-video synchronization is enabled + DHLOGD("OnSmoothFinished rawTime: %{public}ld, isAVsync: %{public}d", + buffer->frameInfo_.rawTime, workModeParam_.isAVsync); + if (workModeParam_.isAVsync) { + std::lock_guard lock(syncBufferMutex_); + if (syncBufferQueue_.size() >= DCAMERA_MAX_SYNC_BUFFER_SIZE) { + DHLOGI("Sync buffer full, drop oldest frame, streamId: %{public}d", streamId_); + syncBufferQueue_.pop_front(); + } + syncBufferQueue_.push_back(buffer); + syncBufferCond_.notify_one(); // Notify the synchronization thread to process + return ; + } auto feedFunc = [this, dhBase, buffer]() { FeedStreamToDriver(dhBase, buffer); }; @@ -287,5 +318,178 @@ void DCameraStreamDataProcessProducer::OnSmoothFinished(const std::shared_ptrPostTask(feedFunc); } } + +void DCameraStreamDataProcessProducer::SyncVideoThread() +{ + DHLOGI("SyncVideoThread started for streamId: %{public}d", streamId_); + const std::chrono::milliseconds FRAME_INTERVAL(DCAMERA_SYNC_TIME_INTERVAL); // 33ms per frame + std::chrono::steady_clock::time_point nextScheduleTime; + + while (syncRunning_.load()) { + std::shared_ptr buffer = nullptr; + + // Wait for video frames in the sync queue + bool shouldBreak = WaitForVideoFrame(buffer); + if (shouldBreak) { + break; + } + + if (buffer == nullptr) { + continue; + } + + // Record the start time when the first frame is sent + if (isFirstFrame_.load()) { + nextScheduleTime = std::chrono::steady_clock::now(); + isFirstFrame_.store(false); + DHLOGI("First frame timestamp recorded, streamId: %{public}d", streamId_); + } + + // Synchronization frame judgment + uint64_t videoPtsUs = buffer->frameInfo_.rawTime; + int32_t syncResult = SyncVideoFrame(videoPtsUs); + if (syncResult == 1) { + // Synchronization successful, sending video frame + DHLOGI("Video frame in sync range, sending..."); + DHBase dhBase; + dhBase.deviceId_ = devId_; + dhBase.dhId_ = dhId_; + + int32_t ret = FeedStreamToDriver(dhBase, buffer); + if (ret != DCAMERA_OK) { + DHLOGE("FeedStreamToDriver failed, ret: %{public}d, streamId: %{public}d", ret, streamId_); + } else { + // FeedStreamToDriver success video_update_clock + DHLOGI("FeedStreamToDriver success, streamId: %{public}d", streamId_); + UpdateVideoClock(videoPtsUs); + } + nextScheduleTime += FRAME_INTERVAL; // Perform timed scheduling after successful transmission + std::this_thread::sleep_until(nextScheduleTime); + } else if (syncResult == 0) { + DHLOGI("Video frame too early, rescheduling for next cycle..."); + { + std::lock_guard lock(syncBufferMutex_); + syncBufferQueue_.push_front(buffer); + } + + // Perform timed scheduling after successful transmission + nextScheduleTime += FRAME_INTERVAL; + std::this_thread::sleep_until(nextScheduleTime); + } else { + // Video frame is too late, discard directly and process next frame immediately + DHLOGI("Video frame too late, dropped..."); + continue; + } + } + + DHLOGI("SyncVideoThread exited for streamId: %{public}d", streamId_); +} + +bool DCameraStreamDataProcessProducer::WaitForVideoFrame(std::shared_ptr& buffer) +{ + std::unique_lock lock(syncBufferMutex_); + syncBufferCond_.wait(lock, [this] { + return !syncBufferQueue_.empty() || !syncRunning_.load(); + }); + + if (!syncRunning_.load()) { + return true; // exit + } + + if (!syncBufferQueue_.empty() && syncBufferQueue_.size() >= DCAMERA_SYNC_WATERMARK) { + buffer = syncBufferQueue_.front(); + syncBufferQueue_.pop_front(); + } + + return false; // don't exit +} + +int32_t DCameraStreamDataProcessProducer::SyncVideoFrame(uint64_t videoPtsUs) +{ + int64_t videoPts = videoPtsUs / DCAMERA_NS_TO_MS; // us -> ms + int64_t audioPts = 0; // get audio timestamp from shared memory + int64_t audioUpdatePts = 0; // audio update time + float audioSpeed = 1.0f; // audio playback speed factor + bool ret = false; + if (syncMem_ == nullptr) { + syncMem_ = sptr(new (std::nothrow) Ashmem(workModeParam_.fd, workModeParam_.sharedMemLen)); + CHECK_AND_RETURN_RET_LOG(syncMem_ == nullptr, DCAMERA_BAD_VALUE, "SyncVideoFrame: syncMem_ is nullptr"); + + ret = syncMem_->MapReadAndWriteAshmem(); + CHECK_AND_RETURN_RET_LOG(!ret, DCAMERA_BAD_VALUE, "SyncVideoFrame: MapReadAndWriteAshmem failed"); + } else { + DHLOGI("SyncVideoFrame: syncMem_ is already init"); + } + auto syncData = syncMem_->ReadFromAshmem(workModeParam_.sharedMemLen, 0); + SyncSharedData *readSyncSharedData = reinterpret_cast(const_cast(syncData)); + CHECK_AND_RETURN_RET_LOG(readSyncSharedData == nullptr, DCAMERA_BAD_VALUE, "read SyncData failed"); + // get voliate lock + while (!readSyncSharedData->lock) { + DHLOGI("readSyncSharedData->lock is false"); + syncData = syncMem_->ReadFromAshmem(workModeParam_.sharedMemLen, 0); + readSyncSharedData = reinterpret_cast(const_cast(syncData)); + } + readSyncSharedData->lock = 0; + ret = syncMem_->WriteToAshmem(static_cast(readSyncSharedData), sizeof(SyncSharedData), 0); + CHECK_AND_RETURN_RET_LOG(!ret, DCAMERA_BAD_VALUE, "write sync data failed!"); + audioPts = readSyncSharedData->audio_current_pts / DCAMERA_US_TO_MS; // us -> ms + audioUpdatePts = readSyncSharedData->audio_update_clock / DCAMERA_US_TO_MS; // us -> ms + audioSpeed = readSyncSharedData->audio_speed; + readSyncSharedData->lock = 1; + ret = syncMem_->WriteToAshmem(static_cast(readSyncSharedData), sizeof(SyncSharedData), 0); + CHECK_AND_RETURN_RET_LOG(!ret, DCAMERA_BAD_VALUE, "write sync data failed!"); + + int64_t currentTime = static_cast(GetNowTimeStampMs()); + int64_t estimatedPts = static_cast(audioPts + (currentTime - audioUpdatePts) * audioSpeed); + + int64_t diff = static_cast(estimatedPts - videoPts); // calculate audio-video time difference + DHLOGD("SyncCheck: videoPts=%{public}" PRIu64 ", estimatedPts=%{public}" PRIu64 ", diff=%{public}" PRId64 "ms", + videoPts, estimatedPts, diff); + + if (diff > DCAMERA_TIME_DIFF_MAX) { + DHLOGI("Video is too late (diff=%{public}" PRId64 "ms), skip this frame.", diff); + // Drop if there is still data in the queue, play the last frame directly + return (syncBufferQueue_.size() > 0) ? -1 : 1; + } else if (diff < DCAMERA_TIME_DIFF_MIN) { + DHLOGI("Video is too early (diff=%{public}" PRId64 "ms), wait for next scheduling.", diff); + return 0; + } else { + DHLOGD("Video frame in sync range, will be sent. diff=%{public}" PRId64 "ms", diff); + return 1; + } +} + +void DCameraStreamDataProcessProducer::UpdateVideoClock(uint64_t videoPtsUs) +{ + if (!workModeParam_.isAVsync) { + return; + } + + CHECK_AND_RETURN_LOG(syncMem_ == nullptr, "UpdateVideoClock: syncMem_ is nullptr."); + + auto syncData = syncMem_->ReadFromAshmem(workModeParam_.sharedMemLen, 0); + SyncSharedData *readSyncSharedData = reinterpret_cast(const_cast(syncData)); + CHECK_AND_RETURN_LOG(readSyncSharedData == nullptr, "read SyncData failed"); + while (!readSyncSharedData->lock) { + syncData = syncMem_->ReadFromAshmem(workModeParam_.sharedMemLen, 0); + readSyncSharedData = reinterpret_cast(const_cast(syncData)); + } + readSyncSharedData->lock = 0; + bool ret = syncMem_->WriteToAshmem(static_cast(readSyncSharedData), sizeof(SyncSharedData), 0); + CHECK_AND_RETURN_LOG(!ret, "write sync data failed!"); + readSyncSharedData->video_current_pts = videoPtsUs; + readSyncSharedData->video_update_clock = GetNowTimeStampUs(); + readSyncSharedData->reset = false; + readSyncSharedData->lock = 1; + ret = syncMem_->WriteToAshmem(static_cast(readSyncSharedData), sizeof(SyncSharedData), 0); + CHECK_AND_RETURN_LOG(!ret, "write sync data failed!"); + DHLOGD("Video update clock updated to: %" PRIu64 " us", readSyncSharedData->video_update_clock); +} + +void DCameraStreamDataProcessProducer::UpdateProducerWorkMode(const WorkModeParam& param) +{ + DHLOGI("Update DistributedHardware WorkMode"); + workModeParam_ = param; +} } // namespace DistributedHardware } // namespace OHOS diff --git a/services/cameraservice/sourceservice/test/unittest/common/distributedcameramgr/dcamera_stream_data_process_producer_test.cpp b/services/cameraservice/sourceservice/test/unittest/common/distributedcameramgr/dcamera_stream_data_process_producer_test.cpp index bb6fe7c5719825740ad21d220ac3615232b5d98a..86562daa6ffe288aed803bf98a7f4e5f8340e825 100644 --- a/services/cameraservice/sourceservice/test/unittest/common/distributedcameramgr/dcamera_stream_data_process_producer_test.cpp +++ b/services/cameraservice/sourceservice/test/unittest/common/distributedcameramgr/dcamera_stream_data_process_producer_test.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #define private public #include "dcamera_stream_data_process_producer.h" #undef private @@ -24,6 +25,7 @@ #include "distributed_camera_constants.h" #include "distributed_camera_errno.h" #include "distributed_hardware_log.h" +#include "idistributed_camera_source.h" #include "v1_1/dcamera_types.h" using namespace testing::ext; @@ -36,6 +38,12 @@ public: static void TearDownTestCase(void); void SetUp(); void TearDown(); + + std::shared_ptr producer_ = nullptr; + std::string devId_ = "test_devId"; + std::string dhId_ = "test_dhId"; + int32_t streamId_ = 1; + DCStreamType streamType_ = CONTINUOUS_FRAME; }; namespace { @@ -44,22 +52,32 @@ const std::string TEST_CAMERA_DH_ID_0 = "camera_0"; const uint8_t SLEEP_TIME = 1; const int32_t STREAM_ID_1 = 1; const int32_t STREAM_ID_2 = 2; -#define DCAMERA_PRODUCER_MAX_BUFFER_SIZE 2 } void DCameraStreamDataProcessProducerTest::SetUpTestCase(void) { + DHLOGI("DCameraStreamDataProcessProducerTest SetUpTestCase"); } void DCameraStreamDataProcessProducerTest::TearDownTestCase(void) { + DHLOGI("DCameraStreamDataProcessProducerTest TearDownTestCase"); } void DCameraStreamDataProcessProducerTest::SetUp(void) { + DHLOGI("DCameraStreamDataProcessProducerTest SetUp"); + producer_ = std::make_shared(devId_, dhId_, streamId_, streamType_); } void DCameraStreamDataProcessProducerTest::TearDown(void) { + DHLOGI("DCameraStreamDataProcessProducerTest TearDown"); + if (producer_ != nullptr) { + if (producer_->state_ == DCameraStreamDataProcessProducer::DCAMERA_PRODUCER_STATE_START) { + producer_->Stop(); + } + producer_ = nullptr; + } } /** @@ -228,5 +246,476 @@ HWTEST_F(DCameraStreamDataProcessProducerTest, dcamera_stream_data_process_produ ret = streamProcess2->FeedStreamToDriver(dhBase, buffer); EXPECT_EQ(DCAMERA_BAD_VALUE, ret); } + +/** + * @tc.name: Start_001 + * @tc.desc: Verify DCameraStreamDataProcessProducer Start with CONTINUOUS_FRAME stream type. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, Start_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest Start_001"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = CONTINUOUS_FRAME; + producer_->Start(); + EXPECT_EQ(DCameraStreamDataProcessProducer::DCAMERA_PRODUCER_STATE_START, producer_->state_); +} + +/** + * @tc.name: Start_002 + * @tc.desc: Test DCameraStreamDataProcessProducer Start with SNAPSHOT_FRAME stream type. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, Start_002, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest Start_002"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = SNAPSHOT_FRAME; + producer_->Start(); + EXPECT_EQ(DCameraStreamDataProcessProducer::DCAMERA_PRODUCER_STATE_START, producer_->state_); +} + +/** + * @tc.name: Stop_001 + * @tc.desc: Test DCameraStreamDataProcessProducer Stop when in start state with CONTINUOUS_FRAME stream type. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, Stop_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest Stop_001"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = CONTINUOUS_FRAME; + producer_->Start(); + EXPECT_EQ(DCameraStreamDataProcessProducer::DCAMERA_PRODUCER_STATE_START, producer_->state_); + producer_->Stop(); + EXPECT_EQ(DCameraStreamDataProcessProducer::DCAMERA_PRODUCER_STATE_STOP, producer_->state_); +} + +/** + * @tc.name: Stop_002 + * @tc.desc: Test DCameraStreamDataProcessProducer Stop when in start state with SNAPSHOT_FRAME stream type. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, Stop_002, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest Stop_002"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = SNAPSHOT_FRAME; + producer_->Start(); + EXPECT_EQ(DCameraStreamDataProcessProducer::DCAMERA_PRODUCER_STATE_START, producer_->state_); + producer_->Stop(); + EXPECT_EQ(DCameraStreamDataProcessProducer::DCAMERA_PRODUCER_STATE_STOP, producer_->state_); +} + +/** + * @tc.name: FeedStream_001 + * @tc.desc: Test DCameraStreamDataProcessProducer FeedStream with CONTINUOUS_FRAME stream type. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, FeedStream_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest FeedStream_001"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = CONTINUOUS_FRAME; + producer_->Start(); + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1000000; + producer_->FeedStream(buffer); + producer_->Stop(); + SUCCEED(); +} + +/** + * @tc.name: FeedStream_002 + * @tc.desc: Test DCameraStreamDataProcessProducer FeedStream with SNAPSHOT_FRAME stream type. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, FeedStream_002, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest FeedStream_002"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = SNAPSHOT_FRAME; + producer_->Start(); + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1000000; + producer_->FeedStream(buffer); + { + std::lock_guard lock(producer_->bufferMutex_); + EXPECT_FALSE(producer_->buffers_.empty()); + } + producer_->Stop(); +} + +/** + * @tc.name: FeedStream_003 + * @tc.desc: Test DCameraStreamDataProcessProducer FeedStream when buffer queue is full. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, FeedStream_003, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest FeedStream_003"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = SNAPSHOT_FRAME; + producer_->Start(); + + for (uint32_t i = 0; i <= producer_->DCAMERA_PRODUCER_MAX_BUFFER_SIZE; i++) { + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1000000 + i * 1000; // Incremental timestamps + producer_->FeedStream(buffer); + } + + { + std::lock_guard lock(producer_->bufferMutex_); + EXPECT_EQ(producer_->buffers_.size(), producer_->DCAMERA_PRODUCER_MAX_BUFFER_SIZE); + } + producer_->Stop(); +} + +/** + * @tc.name: OnSmoothFinished_001 + * @tc.desc: Test DCameraStreamDataProcessProducer OnSmoothFinished with AV sync disabled. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, OnSmoothFinished_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest OnSmoothFinished_001"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = CONTINUOUS_FRAME; + producer_->DCameraStreamDataProcessProducer::workModeParam_.isAVsync = false; + producer_->Start(); + + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1000000; // 1s in us + auto feedableData = std::static_pointer_cast(buffer); + producer_->OnSmoothFinished(feedableData); + + producer_->Stop(); + SUCCEED(); +} + +/** + * @tc.name: OnSmoothFinished_002 + * @tc.desc: Test DCameraStreamDataProcessProducer OnSmoothFinished with AV sync enabled. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, OnSmoothFinished_002, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest OnSmoothFinished_002"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = CONTINUOUS_FRAME; + producer_->DCameraStreamDataProcessProducer::workModeParam_.isAVsync = true; + producer_->Start(); + + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1000000; // 1s in us + auto feedableData = std::static_pointer_cast(buffer); + producer_->OnSmoothFinished(feedableData); + + { + std::lock_guard lock(producer_->syncBufferMutex_); + EXPECT_FALSE(producer_->syncBufferQueue_.empty()); + } + producer_->Stop(); +} + +/** + * @tc.name: OnSmoothFinished_003 + * @tc.desc: Test DCameraStreamDataProcessProducer OnSmoothFinished with full sync buffer queue. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, OnSmoothFinished_003, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest OnSmoothFinished_003"); + ASSERT_NE(producer_, nullptr); + producer_->streamType_ = CONTINUOUS_FRAME; + producer_->DCameraStreamDataProcessProducer::workModeParam_.isAVsync = true; + producer_->Start(); + + for (uint32_t i = 0; i <= producer_->DCAMERA_MAX_SYNC_BUFFER_SIZE; i++) { + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1000000 + i * 1000; // Incremental timestamps + auto feedableData = std::static_pointer_cast(buffer); + producer_->OnSmoothFinished(feedableData); + } + + { + std::lock_guard lock(producer_->syncBufferMutex_); + EXPECT_EQ(producer_->syncBufferQueue_.size(), producer_->DCAMERA_MAX_SYNC_BUFFER_SIZE); + } + producer_->Stop(); +} + +/** + * @tc.name: UpdateProducerWorkMode_001 + * @tc.desc: Test DCameraStreamDataProcessProducer UpdateProducerWorkMode. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, UpdateProducerWorkMode_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest UpdateProducerWorkMode_001"); + ASSERT_NE(producer_, nullptr); + WorkModeParam param(-1, 0, 0, 0); + param.fd = 10; + param.sharedMemLen = 1024; + param.isAVsync = true; + param.scene = 0; + + producer_->UpdateProducerWorkMode(param); + uint64_t videoPtsUs = 1000000; + int32_t result = producer_->SyncVideoFrame(videoPtsUs); + EXPECT_TRUE(result == DCAMERA_BAD_VALUE || result == -1 || result == 0 || result == 1); +} + +/** + * @tc.name: SyncVideoFrame_001 + * @tc.desc: Test DCameraStreamDataProcessProducer SyncVideoFrame with null syncMem. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, SyncVideoFrame_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest SyncVideoFrame_001"); + ASSERT_NE(producer_, nullptr); + uint64_t videoPtsUs = 1000000; // 1s in us + int32_t result = producer_->SyncVideoFrame(videoPtsUs); + EXPECT_TRUE(result == DCAMERA_BAD_VALUE || result == -1 || result == 0 || result == 1); +} + +/** + * @tc.name: SyncVideoFrame_002 + * @tc.desc: Verify SyncVideoFrame function with different timestamp values. + * @tc.type: FUNC + * @tc.require: Issue Number + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, SyncVideoFrame_002, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest SyncVideoFrame_002"); + ASSERT_NE(producer_, nullptr); + + WorkModeParam param(-1, 0, 0, 0); + param.fd = 10; + param.sharedMemLen = 1024; + param.isAVsync = 1; + param.scene = 0; + producer_->UpdateProducerWorkMode(param); + + uint64_t videoPtsUs1 = 1000000; // 1 second + uint64_t videoPtsUs2 = 50000000; // 50 seconds + uint64_t videoPtsUs3 = 100000; // 0.1 seconds + + int32_t result1 = producer_->SyncVideoFrame(videoPtsUs1); + int32_t result2 = producer_->SyncVideoFrame(videoPtsUs2); + int32_t result3 = producer_->SyncVideoFrame(videoPtsUs3); + + EXPECT_TRUE(result1 == DCAMERA_BAD_VALUE || result1 == -1 || result1 == 0 || result1 == 1); + EXPECT_TRUE(result2 == DCAMERA_BAD_VALUE || result2 == -1 || result2 == 0 || result2 == 1); + EXPECT_TRUE(result3 == DCAMERA_BAD_VALUE || result3 == -1 || result3 == 0 || result3 == 1); +} + +/** + * @tc.name: SyncVideoFrame_003 + * @tc.desc: Verify SyncVideoFrame function when AV sync is disabled. + * @tc.type: FUNC + * @tc.require: Issue Number + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, SyncVideoFrame_003, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest SyncVideoFrame_003"); + ASSERT_NE(producer_, nullptr); + + WorkModeParam param(-1, 0, 0, 0); + param.fd = 10; + param.sharedMemLen = 1024; + param.isAVsync = 0; // AV sync disabled + param.scene = 0; + producer_->UpdateProducerWorkMode(param); + + uint64_t videoPtsUs = 1000000; + int32_t result = producer_->SyncVideoFrame(videoPtsUs); + + EXPECT_TRUE(result == DCAMERA_BAD_VALUE || result == -1 || result == 0 || result == 1); +} + +/** + * @tc.name: SyncVideoFrame_004 + * @tc.desc: Integration test for SyncVideoFrame through public interface. + * @tc.type: FUNC + * @tc.require: Issue Number + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, SyncVideoFrame_004, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest SyncVideoFrame_004"); + ASSERT_NE(producer_, nullptr); + + WorkModeParam param(-1, 0, 0, 0); + param.fd = 10; + param.sharedMemLen = 1024; + param.isAVsync = 1; + param.scene = 0; + producer_->UpdateProducerWorkMode(param); + + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1000000; // Set a specific timestamp + + producer_->Start(); + + producer_->FeedStream(buffer); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + // Stop the producer + producer_->Stop(); + SUCCEED(); +} + +/** + * @tc.name: UpdateVideoClock_001 + * @tc.desc: Verify UpdateVideoClock function when AV sync is disabled. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, UpdateVideoClock_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest UpdateVideoClock_001"); + ASSERT_NE(producer_, nullptr); + uint64_t videoPtsUs = 1000000; // 1s in us + WorkModeParam param(-1, 0, 0, 0); + param.isAVsync = 0; + producer_->DCameraStreamDataProcessProducer::workModeParam_ = param; + + producer_->UpdateVideoClock(videoPtsUs); + SUCCEED(); +} + +/** + * @tc.name: UpdateVideoClock_002 + * @tc.desc: Verify UpdateVideoClock function when syncMem is null. + * @tc.type: FUNC + * @tc.require: Issue Number + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, UpdateVideoClock_002, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest UpdateVideoClock_002"); + ASSERT_NE(producer_, nullptr); + + WorkModeParam param(-1, 0, 0, 0); + param.isAVsync = 1; + producer_->DCameraStreamDataProcessProducer::workModeParam_ = param; + producer_->syncMem_ = nullptr; + + uint64_t videoPtsUs = 1000000; + producer_->UpdateVideoClock(videoPtsUs); + + SUCCEED(); +} + +/** + * @tc.name: UpdateVideoClock_003 + * @tc.desc: Verify UpdateVideoClock function updates video clock correctly. + * @tc.type: FUNC + * @tc.require: Issue Number + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, UpdateVideoClock_003, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest UpdateVideoClock_003"); + ASSERT_NE(producer_, nullptr); + + WorkModeParam param(-1, 0, 0, 0); + param.isAVsync = 1; + param.fd = 10; + param.sharedMemLen = 1024; + producer_->DCameraStreamDataProcessProducer::workModeParam_ = param; + + producer_->syncMem_ = sptr(new (std::nothrow) Ashmem(param.fd, param.sharedMemLen)); + + uint64_t videoPtsUs = 2000000; + producer_->UpdateVideoClock(videoPtsUs); + + SUCCEED(); +} + +/** + * @tc.name: UpdateVideoClock_004 + * @tc.desc: Integration test for UpdateVideoClock through FeedStream. + * @tc.type: FUNC + * @tc.require: Issue Number + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, UpdateVideoClock_004, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest UpdateVideoClock_004"); + ASSERT_NE(producer_, nullptr); + + WorkModeParam param(-1, 0, 0, 0); + param.isAVsync = 1; + param.fd = 10; + param.sharedMemLen = 1024; + param.scene = 0; + producer_->UpdateProducerWorkMode(param); + + producer_->Start(); + + auto buffer = std::make_shared(100); + buffer->frameInfo_.rawTime = 1500000; // Set a specific timestamp + + producer_->FeedStream(buffer); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + producer_->Stop(); + SUCCEED(); +} + +/** + * @tc.name: UpdateVideoClock_005 + * @tc.desc: Verify UpdateVideoClock function handles different video timestamps. + * @tc.type: FUNC + * @tc.require: Issue Number + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, UpdateVideoClock_005, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest UpdateVideoClock_005"); + ASSERT_NE(producer_, nullptr); + + WorkModeParam param(-1, 0, 0, 0); + param.isAVsync = 1; + param.fd = 10; + param.sharedMemLen = 1024; + producer_->DCameraStreamDataProcessProducer::workModeParam_ = param; + + producer_->syncMem_ = sptr(new (std::nothrow) Ashmem(param.fd, param.sharedMemLen)); + std::vector timestamps = {0, 1000000, 50000000, 100000000}; + for (uint64_t timestamp : timestamps) { + producer_->UpdateVideoClock(timestamp); + } + + SUCCEED(); +} + +/** + * @tc.name: WaitForVideoFrame_001 + * @tc.desc: Test DCameraStreamDataProcessProducer WaitForVideoFrame with empty queue and stopped sync. + * @tc.type: FUNC + * @tc.require: issue + */ +HWTEST_F(DCameraStreamDataProcessProducerTest, WaitForVideoFrame_001, TestSize.Level1) +{ + DHLOGI("DCameraStreamDataProcessProducerTest WaitForVideoFrame_001"); + std::shared_ptr buffer = nullptr; + producer_->syncRunning_.store(false); + + bool shouldBreak = producer_->WaitForVideoFrame(buffer); + EXPECT_TRUE(shouldBreak); + EXPECT_EQ(buffer, nullptr); +} } } diff --git a/services/channel/src/dcamera_softbus_adapter.cpp b/services/channel/src/dcamera_softbus_adapter.cpp index 0dbe8f5a013ea51374c8331099f51e3f3980f697..8f726ade9fc26b10d3d890307c69d79d93c7f421 100644 --- a/services/channel/src/dcamera_softbus_adapter.cpp +++ b/services/channel/src/dcamera_softbus_adapter.cpp @@ -324,6 +324,7 @@ int32_t DCameraSoftbusAdapter::SendSofbusStream(int32_t socket, std::shared_ptr< sinkFrameInfo.startEncodeT_ = startEncodeT; sinkFrameInfo.finishEncodeT_ = finishEncodeT; sinkFrameInfo.sendT_ = GetNowTimeStampUs(); + sinkFrameInfo.rawTime_ = std::to_string(timeStamp); sinkFrameInfo.Marshal(jsonStr); StreamData ext = { const_cast(jsonStr.c_str()), jsonStr.length() }; StreamFrameInfo param = { 0 }; @@ -489,6 +490,11 @@ int32_t DCameraSoftbusAdapter::HandleSourceStreamExt(std::shared_ptr frameInfo.pts = sinkFrameInfo.pts_; frameInfo.index = sinkFrameInfo.index_; frameInfo.ver = sinkFrameInfo.ver_; + if (sinkFrameInfo.rawTime_.empty()) { + DHLOGI("sinkFrameInfo rawTime_ is empty."); + } else { + frameInfo.rawTime = std::stoll(sinkFrameInfo.rawTime_); + } frameInfo.timePonit.startEncode = sinkFrameInfo.startEncodeT_; frameInfo.timePonit.finishEncode = sinkFrameInfo.finishEncodeT_; frameInfo.timePonit.send = sinkFrameInfo.sendT_; diff --git a/services/data_process/src/pipeline_node/multimedia_codec/encoder/encode_data_process.cpp b/services/data_process/src/pipeline_node/multimedia_codec/encoder/encode_data_process.cpp index d78ec93361a73d4677fec37059ecc17fd248836d..79a2ccce13582ddece5aa92dd076327cbbb87958 100644 --- a/services/data_process/src/pipeline_node/multimedia_codec/encoder/encode_data_process.cpp +++ b/services/data_process/src/pipeline_node/multimedia_codec/encoder/encode_data_process.cpp @@ -497,9 +497,9 @@ int32_t EncodeDataProcess::GetEncoderOutputBuffer(uint32_t index, MediaAVCodec:: CHECK_AND_RETURN_RET_LOG(err != EOK, DCAMERA_MEMORY_OPT_ERROR, "%{public}s", "memcpy_s buffer failed."); int64_t timeStamp = info.presentationTimeUs; struct timespec time = {0, 0}; - clock_gettime(CLOCK_MONOTONIC, &time); + clock_gettime(CLOCK_REALTIME, &time); int64_t timeNs = static_cast(time.tv_sec) * S2NS + static_cast(time.tv_nsec); - int64_t encodeT = (timeNs - timeStamp) / static_cast(US2NS); + int64_t encodeT = timeNs / static_cast(US2NS) - timeStamp; int64_t finishEncodeT = GetNowTimeStampUs(); int64_t startEncodeT = finishEncodeT - encodeT; bufferOutput->SetInt64(START_ENCODE_TIME_US, startEncodeT);