From 30239ca93b1769fedb6eb609d03da37d712a051d Mon Sep 17 00:00:00 2001 From: byndyx Date: Tue, 10 Dec 2024 15:10:18 +0800 Subject: [PATCH] encoder filter Signed-off-by: byndyx --- .../filters/av_transport_coder/BUILD.GN | 21 + .../av_trans_audio_encoder_filter.cpp | 643 ++++++++++++++++++ .../av_trans_audio_encoder_filter.h | 178 +++++ .../av_trans_video_encoder_filter.cpp | 643 ++++++++++++++++++ .../av_trans_video_encoder_filter.h | 151 ++++ 5 files changed, 1636 insertions(+) create mode 100644 av_transport/av_trans_engine/filters/av_transport_coder/BUILD.GN create mode 100644 av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.cpp create mode 100644 av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.h create mode 100644 av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.cpp create mode 100644 av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.h diff --git a/av_transport/av_trans_engine/filters/av_transport_coder/BUILD.GN b/av_transport/av_trans_engine/filters/av_transport_coder/BUILD.GN new file mode 100644 index 00000000..b701a940 --- /dev/null +++ b/av_transport/av_trans_engine/filters/av_transport_coder/BUILD.GN @@ -0,0 +1,21 @@ +audio_encoder: + + external_deps += [ + "av_codec:av_codec_client", + "av_codec:native_media_acodec", + "media_foundation:media_foundation", + "media_foundation:native_media_core", + ] + + + cJSON: + + av_codec + "deps": { + "components": [ + "av_codec", + +video_encoder: + + + diff --git a/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.cpp b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.cpp new file mode 100644 index 00000000..e6542a1d --- /dev/null +++ b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.cpp @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "av_transport_encoder_filter.h" + +#include "av_trans_log.h" +#include "filter_factory.h" + +#undef DH_LOG_TAG +#define DH_LOG_TAG "AudioEncoderFilter" + +namespace OHOS { +namespace DistributedHardware { +namespace Pipeline { +static AutoRegisterFilterg_registerAudioEncoderFilter("builtin.recorder.audioencoderfilter", + FilterType::FILTERTYPE_AENC, + [](const std::string& name, const FilterType type) { + return std::make_shared(name, FilterType::FILTERTYPE_AENC); + }); + +class AudioEncoderFilterCallback : public FilterLinkCallback { +public: + explicit AudioEncoderFilterCallback(std::shared_ptr filter) + : inputFilter_(std::move(filter)) {} + ~AudioEncoderFilterCallback() = default; + + void OnLinkedResult(const sptr& queue, + std::shared_ptr& meta) override + { + if (auto filter = inputFilter_.lock()) { + filter->OnLinkedResult(queue, meta); + } else { + AVTRANS_LOGE("Invalid inputfilter"); + } + } + void OnUnlinkedResult(std::shared_ptr& meta) override + { + if (auto filter = inputFilter_.lock()) { + filter->OnUnLinkedResult(meta); + } else { + AVTRANS_LOGE("Invalid inputfilter"); + } + } + void OnUpdatedResult(std::shared_ptr& meta) override + { + if (auto filter = inputFilter_.lock()) { + filter->OnUpdatedResult(meta); + } else { + AVTRANS_LOGE("Invalid inputfilter"); + } + } +private: + std::weak_ptr inputFilter_ {}; +}; + +class AVBufferAvaliableListener : public Media::IConsumerListener { +public: + AVBufferAvaliableListener(std::weak_ptr encoder) + { + encoder_ = encoder; + } + void OnBufferAvailable() override + { + auto encoder = encoder_.lock(); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->ProcessInputBuffer(); + } +private: + std::weak_ptr encoder_; +}; + +static void OnError(OH_AVCodec *codec, int32_t errorCode, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncError(errorCode); +} + +static void OnOutputFormatChanged(OH_AVCodec *codec, OH_AVFormat *format, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncOutputFormatChanged(format); +} + +static void OnInputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncInputBufferAvailable(index, buffer); +} + +static void OnOutputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncOutputBufferAvailable(index, buffer); +} + +AudioEncoderFilter::AudioEncoderFilter(std::string name, FilterType type) : Filter(name, type) +{ +} + +AudioEncoderFilter::~AudioEncoderFilter() +{ + AVTRANS_LOGI("enter"); + if (audioEncoder_ != nullptr) { + AVTRANS_LOGI("release audio encoder"); + ReleaseAudioCodec(); + } +} + +void AudioEncoderFilter::Init(const std::shared_ptr& receiver, const std::shared_ptr& callback) +{ + receiver_ = receiver; + callback_ = callback; +} + +Status AudioEncoderFilter::DoInitAfterLink() +{ + return Status::OK; +} + +Status AudioEncoderFilter::PrepareInputBufferQueue() +{ + AVTRANS_LOGI("enter"); + if (inputBufferQueue_ == nullptr) { + inputBufferQueue_ = Media::AVBufferQueue::Create(DEFAULT_BUFFER_NUM, Media::MemoryType::VIRTUAL_MEMORY, + INPUT_BUFFERQUEUE_NAME); + } + TRUE_RETURN_V_MSG_E(inputBufferQueue_ == nullptr, Status::ERROR_NULL_POINTER, "create bufferqueue failed"); + inputProducer_ = inputBufferQueue_->GetProducer(); + TRUE_RETURN_V_MSG_E(inputProducer_ == nullptr, Status::ERROR_NULL_POINTER, "GetProducer failed"); + inputConsumer_ = inputBufferQueue_->GetConsumer(); + TRUE_RETURN_V_MSG_E(inputConsumer_ == nullptr, Status::ERROR_NULL_POINTER, "GetConsumer failed"); + + sptr listener(new AVBufferAvaliableListener(shared_from_this())); + inputConsumer_->SetBufferAvailableListener(listener); + return Status::OK; +} + +Status AudioEncoderFilter::DoPrepare() +{ + AVTRANS_LOGI("enter"); + auto ret = PrepareInputBufferQueue(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "PrepareInputBufferQueue failed"); + + ret = CreateAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "Create AudioCodec failed"); + ret = ConfigureAudioCodec(initEncParams_); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "Configure AudioCodec failed"); + + TRUE_RETURN_V_MSG_E(callback_ == nullptr, Status::ERROR_NULL_POINTER, "callback is nullptr"); + callback_->OnCallback(shared_from_this(), FilterCallBackCommand::NEXT_FILTER_NEEDED, + StreamType::STREAMTYPE_RAW_AUDIO); + return Status::OK; +} + +Status AudioEncoderFilter::DoStart() +{ + AVTRANS_LOGI("enter"); + auto ret = StartAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "StartAudioCodec failed"); + return Status::OK; +} + +Status AudioEncoderFilter::DoPause() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoPauseDragging() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoResume() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoResumeDragging() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoStop() +{ + AVTRANS_LOGI("enter"); + auto ret = StopAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "StopAudioCodec failed"); + return Status::OK; +} + +Status AudioEncoderFilter::DoFlush() +{ + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + auto res = OH_AudioCodec_Flush(audioEncoder_); + TRUE_RETURN_V_MSG_E(res != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "audioEncoder flush failed: %{public}d", res); + return Status::OK; +} + +Status AudioEncoderFilter::DoRelease() +{ + AVTRANS_LOGI("enter"); + auto ret = ReleaseAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "StartAudioCodec failed"); + return Status::OK; +} + +Status AudioEncoderFilter::DoProcessInputBuffer(int recvArg, bool dropFrame) +{ + AVTRANS_LOGD("enter"); + std::shared_ptr buffer = nullptr; + TRUE_RETURN_V_MSG_E(inputConsumer_ == nullptr, Status::ERROR_NULL_POINTER, "inputConsumer is null"); + Media::Status ret = inputConsumer_->AcquireBuffer(buffer); + TRUE_RETURN_V_MSG_E(ret != Media::Status::OK, Status::ERROR_INVALID_OPERATION, "AcquireBuffer failed"); + { + std::lock_guard datalock(mtxData_); + while (inputDataBufferQueue_.size() > AUDIO_ENCODER_QUEUE_MAX) { + AVTRANS_LOGE("inputDataBufferQueue_ overflow"); + auto frontBuffer = inputDataBufferQueue_.front(); + inputDataBufferQueue_.pop(); + if (frontBuffer != nullptr) { + inputConsumer_->ReleaseBuffer(frontBuffer); + } + } + inputDataBufferQueue_.push(buffer); + } + encodeCond_.notify_all(); + return Status::OK; +} + +Status AudioEncoderFilter::DoProcessOutputBuffer(int recvArg, bool dropFrame, bool byIdx, uint32_t idx, int64_t renderTime) +{ + return Status::OK; +} + +void AudioEncoderFilter::SetParameter(const std::shared_ptr& meta) +{ + encFliterMeta_ = meta; +} + +void AudioEncoderFilter::GetParameter(std::shared_ptr& meta) +{ + meta = encFliterMeta_; +} + +Status AudioEncoderFilter::LinkNext(const std::shared_ptr& nextFilter, StreamType outType) +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(nextFilter == nullptr, Status::ERROR_NULL_POINTER, "input nextFilter is nullptr"); + nextFilter_ = nextFilter; + { + std::lock_guard lock(nextFiltersMutex_); + nextFiltersMap_[outType].push_back(nextFilter); + } + auto filterLinkCallback = std::make_shared(shared_from_this()); + auto ret = nextFilter->OnLinked(outType, configureParam_, filterLinkCallback); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "Onlinked failed: %{public}d", ret); + return Status::OK; +} + +Status AudioEncoderFilter::UpdateNext(const std::shared_ptr& nextFilter, StreamType outType) +{ + return Status::OK; +} + +Status AudioEncoderFilter::UnLinkNext(const std::shared_ptr& nextFilter, StreamType outType) +{ + std::lock_guard lock(nextFiltersMutex_); + nextFiltersMap_.clear(); + return Status::OK; +} + +Status AudioEncoderFilter::OnLinked(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(callback == nullptr, Status::ERROR_NULL_POINTER, "input callback is nullptr"); + onLinkedResultCallback_ = callback; + TRUE_RETURN_V_MSG_E(meta == nullptr, Status::ERROR_NULL_POINTER, "input meta is nullptr"); + meta->GetData(Media::Tag::AUDIO_CHANNEL_COUNT, initEncParams_.channel); + meta->GetData(Media::Tag::AUDIO_SAMPLE_RATE, initEncParams_.sampleRate); + meta->GetData(Media::Tag::AUDIO_SAMPLE_FORMAT, initEncParams_.sampleDepth); // format convert needed ? + auto ret = CheckEncoderFormat(initEncParams_); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "input meta is invalid"); + return Status::OK; +} + +Status AudioEncoderFilter::OnUpdated(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) +{ + return Status::OK; +} + +Status AudioEncoderFilter::OnUnLinked(StreamType inType, const std::shared_ptr& callback) +{ + onLinkedResultCallback_ = nullptr; + return Status::OK; +} + +void AudioEncoderFilter::OnLinkedResult(const sptr& queue, + std::shared_ptr& meta) +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN(queue == nullptr, "input queue is nullptr"); + outputProducer_ = queue; + TRUE_RETURN(onLinkedResultCallback_ == nullptr, "onLinkedResultCallback_ is nullptr"); + onLinkedResultCallback_->OnLinkedResult(inputProducer_, meta); +} + +void AudioEncoderFilter::OnUnLinkedResult(std::shared_ptr& meta) +{ + +} + +void AudioEncoderFilter::OnUpdatedResult(std::shared_ptr& meta) +{ + +} + +Status AudioEncoderFilter::CreateAudioCodec() +{ + AVTRANS_LOGI("enter"); + audioEncoder_ = OH_AudioCodec_CreateByName((MediaAVCodec::AVCodecCodecName::AUDIO_ENCODER_AAC_NAME).data()); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "Create AudioCodec failed"); + OH_AVCodecCallback cb = {&OnError, &OnOutputFormatChanged, &OnInputBufferAvailable, &OnOutputBufferAvailable}; + int32_t ret = OH_AudioCodec_RegisterCallback(audioEncoder_, cb, this); + TRUE_RETURN_V_MSG_E(ret != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, "Codec setCallback failed"); + return Status::OK; +} + +Status AudioEncoderFilter::ConfigureAudioCodec(const AEncInitParams &initEncParams) +{ + AVTRANS_LOGI("enter"); + auto ret = SetEncoderFormat(initEncParams); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "input meta is invalid"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + auto res = OH_AudioCodec_Prepare(audioEncoder_); + TRUE_RETURN_V_MSG_E(res != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "audioEncoder prepare failed: %{public}d", res); + return Status::OK; +} + +Status AudioEncoderFilter::ReleaseAudioCodec() +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + bool isSuccess = true; + if (!isStoped_.load()) { + auto ret = StopAudioCodec(); + if (ret != Status::OK) { + AVTRANS_LOGE("Stop before Release AudioCodec failed"); + isSuccess = false; + } + } + auto res = OH_AudioCodec_Destroy(audioEncoder_); + if (res != AV_ERR_OK) { + AVTRANS_LOGE("Release AudioCodec failed"); + isSuccess = false; + } + audioEncoder_ = nullptr; + return isSuccess ? Status::OK : Status::ERROR_INVALID_OPERATION; +} + +Status AudioEncoderFilter::SetEncoderFormat(const AEncInitParams &initEncParams) +{ + AVTRANS_LOGI("enter"); + auto ret = CheckEncoderFormat(initEncParams); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "input meta is invalid"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + AVTRANS_LOGI("initEncParams channel: %{public}d, sampleRate: %{public}d, sampleDepth: %{public}d", + initEncParams.channel, initEncParams.sampleRate, initEncParams.sampleDepth); + OH_AVFormat *format = OH_AVFormat_Create(); + OH_AVFormat_SetIntValue(format, MediaAVCodec::MediaDescriptionKey::MD_KEY_CHANNEL_COUNT.data(), + initEncParams.channel); + OH_AVFormat_SetIntValue(format, MediaAVCodec::MediaDescriptionKey::MD_KEY_SAMPLE_RATE.data(), + initEncParams.sampleRate); + OH_AVFormat_SetIntValue(format, MediaAVCodec::MediaDescriptionKey::MD_KEY_AUDIO_SAMPLE_FORMAT.data(), + initEncParams.sampleDepth); + int32_t res = OH_AudioCodec_Configure(audioEncoder_, format); + if (res != AV_ERR_OK) { + AVTRANS_LOGE("configure encoder failed: %{public}d", ret); + OH_AVFormat_Destroy(format); + return Status::ERROR_INVALID_OPERATION; + } + OH_AVFormat_Destroy(format); + return Status::OK; +} + +Status AudioEncoderFilter::CheckEncoderFormat(const AEncInitParams &initEncParams) +{ + AVTRANS_LOGI("enter"); + if (initEncParams.channel >= CHANNEL_MASK_MIN && initEncParams.channel <= CHANNEL_MASK_MAX && + initEncParams.sampleRate >= SAMPLE_RATE_MIN && initEncParams.sampleRate <= SAMPLE_RATE_MAX && + initEncParams.sampleDepth == MediaAVCodec::AudioSampleFormat::SAMPLE_S16LE) { + return Status::OK; + } + AVTRANS_LOGE("initEncParams from meta err, channel: %{public}d, sampleRate: %{public}d, sampleDepth: %{public}d", + initEncParams.channel, initEncParams.sampleRate, initEncParams.sampleDepth); + return Status::ERROR_INVALID_PARAMETER; +} + +Status AudioEncoderFilter::StartAudioCodec() +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + auto res = OH_AudioCodec_Start(audioEncoder_); + TRUE_RETURN_V_MSG_E(res != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "audioEncoder start failed: %{public}d", res); + StartInputThread(); + return Status::OK; +} + +Status AudioEncoderFilter::StopAudioCodec() +{ + AVTRANS_LOGI("enter"); + StopInputThread(); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + bool isSuccess = true; + auto ret = OH_AudioCodec_Flush(audioEncoder_); + if (ret != AV_ERR_OK) { + AVTRANS_LOGE("OH_AudioCodec_Flush failed"); + isSuccess = false; + } + TRUE_RETURN_V_MSG_E(OH_AudioCodec_Stop(audioEncoder_)!= AV_ERR_OK || !isSuccess, + Status::ERROR_INVALID_OPERATION, "StopAudioCodec failed"); + firstInputTimeUs_ = 0; + inputTimeStampUs_ = 0; + outputTimeStampUs_ = 0; + waitOutputCount_ = 0; + isStoped_.store(true); + return Status::OK; +} + +void AudioEncoderFilter::StartInputThread() +{ + AVTRANS_LOGI("enter"); + isEncoderRunning_.store(true); + encoderThread_ = std::thread(&AudioEncoderFilter::InputEncodeAudioData, this); + if (pthread_setname_np(encoderThread_.native_handle(), ENCODE_THREAD) != AV_ERR_OK) + { + AVTRANS_LOGE("Encode thread setname failed"); + } +} + +void AudioEncoderFilter::StopInputThread() +{ + AVTRANS_LOGI("enter"); + isEncoderRunning_.store(false); + if (encoderThread_.joinable()) { + encoderThread_.join(); + } + std::lock_guard dataLock(mtxData_); + std::queue().swap(codecIndexQueue_); + std::queue().swap(codecBufQueue_); + while (!inputDataBufferQueue_.empty()) { + auto audioData = inputDataBufferQueue_.front(); + inputDataBufferQueue_.pop(); + inputConsumer_->ReleaseBuffer(audioData); + } +} + +void AudioEncoderFilter::IncreaseWaitEncodeCnt() +{ + std::lock_guard lck(mtxCnt_); + waitOutputCount_++; +} + +void AudioEncoderFilter::ReduceWaitEncodeCnt() +{ + std::lock_guard lck(mtxCnt_); + + if (waitOutputCount_ <= 0) { + AVTRANS_LOGE("waitOutputCount_ is %{public}d", waitOutputCount_); + } + waitOutputCount_--; + AVTRANS_LOGD("waitOutputCount_ is %{public}d", waitOutputCount_); +} + +void AudioEncoderFilter::InputEncodeAudioData() +{ + AVTRANS_LOGD("enter"); + while (isEncoderRunning_.load()) { + std::shared_ptr audioData; + uint32_t index = 0; + OH_AVBuffer *codecMem; + { + std::unique_lock datalock(mtxData_); + encodeCond_.wait_for(datalock, std::chrono::milliseconds(ENCODE_WAIT_MILLISECONDS), + [this]() { + return (!inputDataBufferQueue_.empty() && !codecBufQueue_.empty()); + }); + if (inputDataBufferQueue_.empty() || codecBufQueue_.empty()) { + continue; + } + index = codecIndexQueue_.front(); + codecIndexQueue_.pop(); + codecMem = codecBufQueue_.front(); + codecBufQueue_.pop(); + audioData = inputDataBufferQueue_.front(); + inputDataBufferQueue_.pop(); + } + auto ret = ProcessData(audioData, index, codecMem); + if (ret == Status::ERROR_INVALID_OPERATION) { + AVTRANS_LOGE("Encoder is not runnnig"); + return; + } else if (ret != Status::OK) { + continue; + } + TRUE_RETURN(inputConsumer_ == nullptr, "inputConsumer is null"); + inputConsumer_->ReleaseBuffer(audioData); + } +} + +Status AudioEncoderFilter::ProcessData(std::shared_ptr audioData, const uint32_t index, + OH_AVBuffer *codecMem) +{ + AVTRANS_LOGD("enter"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr || !isEncoderRunning_.load(), Status::ERROR_INVALID_OPERATION, + "Encoder is not runnnig, isEncoderRunning_: %{public}d", isEncoderRunning_.load()); + TRUE_RETURN_V_MSG_E(codecMem == nullptr || codecMem->buffer_ == nullptr || codecMem->buffer_->memory_ == nullptr, + Status::ERROR_NULL_POINTER, "codecMem is invaild"); + TRUE_RETURN_V_MSG_E(audioData == nullptr || audioData->memory_ == nullptr, + Status::ERROR_NULL_POINTER, "audioData is invaild"); + + auto memSize = audioData->memory_->GetSize(); + codecMem->buffer_->memory_->SetSize(memSize); + errno_t err = memcpy_s(OH_AVBuffer_GetAddr(codecMem), memSize, audioData->memory_->GetAddr(), memSize); + TRUE_RETURN_V_MSG_E(err != EOK, Status::ERROR_INVALID_OPERATION, + "memcpy_s err: %{public}d, memSize: %{public}d", err, memSize); + + inputTimeStampUs_ = GetEncoderTimeStamp(); + codecMem->buffer_->pts_ = inputTimeStampUs_; + codecMem->buffer_->flag_ = MediaAVCodec::AVCODEC_BUFFER_FLAG_NONE; + auto ret = OH_AudioCodec_PushInputBuffer(audioEncoder_, index); + TRUE_RETURN_V_MSG_E(ret != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "OH_AudioCodec_PushInputBuffer err: %{public}d", ret); + + IncreaseWaitEncodeCnt(); + return Status::OK; +} + +int64_t AudioEncoderFilter::GetEncoderTimeStamp() +{ + int64_t timeIntervalStampUs = 0; + int64_t nowTimeUs = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + if (firstInputTimeUs_ == 0) { + firstInputTimeUs_ = nowTimeUs; + return timeIntervalStampUs; + } + timeIntervalStampUs = nowTimeUs - firstInputTimeUs_; + return timeIntervalStampUs; +} + +Status AudioEncoderFilter::EncodeDone(AEncOutputData &outputDataEnc) +{ + return Status::OK; +} + +void AudioEncoderFilter::OnEncError(int32_t errorCode) +{ + AVTRANS_LOGE("Encoder err: %{public}d", errorCode); + isEncoderRunning_.store(false); +} + +void AudioEncoderFilter::OnEncOutputFormatChanged(const OH_AVFormat *format) +{ + TRUE_RETURN(format == nullptr, "input format is nullptr"); + outputFormat_.format_ = format->format_; +} + +void AudioEncoderFilter::OnEncInputBufferAvailable(uint32_t index, OH_AVBuffer *buffer) +{ + AVTRANS_LOGD("enter"); + TRUE_RETURN(buffer == nullptr, "input buffer is nullptr"); + { + std::lock_guard datalock(mtxData_); + while (codecBufQueue_.size() > AUDIO_ENCODER_QUEUE_MAX || codecIndexQueue_.size() > AUDIO_ENCODER_QUEUE_MAX) { + AVTRANS_LOGE("codecBufQueue_ or codecIndexQueue_ overflow"); + codecIndexQueue_.pop(); + codecBufQueue_.pop(); + } + codecIndexQueue_.push(index); + codecBufQueue_.push(buffer); + } + encodeCond_.notify_all(); +} + +void AudioEncoderFilter::OnEncOutputBufferAvailable(uint32_t index, OH_AVBuffer *buffer) +{ + AVTRANS_LOGD("enter"); + TRUE_RETURN(audioEncoder_ == nullptr || !isEncoderRunning_.load(), + "Encoder is not runnnig, isEncoderRunning_: %{public}d", isEncoderRunning_.load()); + TRUE_RETURN(buffer == nullptr || buffer->buffer_ == nullptr || buffer->buffer_->memory_ == nullptr || + buffer->buffer_->memory_->GetSize() <= 0, "audioData is invaild"); + TRUE_RETURN(outputProducer_ == nullptr, "input queue is nullptr"); + + Media::AVBufferConfig config; + config.size = buffer->buffer_->memory_->GetSize(); + config.memoryType = Media::MemoryType::VIRTUAL_MEMORY; + config.memoryFlag = Media::MemoryFlag::MEMORY_READ_WRITE; + std::shared_ptr outBuffer = nullptr; + outputProducer_->RequestBuffer(outBuffer, config, MAX_TIME_OUT_MS); + TRUE_RETURN(outBuffer == nullptr || outBuffer->memory_ == nullptr, "RequestBuffer failed"); + auto meta = outBuffer->meta_; + if (meta == nullptr) { + AVTRANS_LOGE("outBuffer->meta_ is null"); + outputProducer_->PushBuffer(outBuffer, true); + return; + } + outBuffer->pts_ = GetEncoderTimeStamp(); + meta->SetData(Media::Tag::USER_FRAME_PTS, outBuffer->pts_); + meta->SetData(Media::Tag::AUDIO_OBJECT_NUMBER, index); + outBuffer->memory_->Write(buffer->buffer_->memory_->GetAddr(), buffer->buffer_->memory_->GetSize(), 0); + outputProducer_->PushBuffer(outBuffer, true); + outputTimeStampUs_ = buffer->buffer_->pts_; + auto ret = OH_AudioCodec_FreeOutputBuffer(audioEncoder_, index); + TRUE_RETURN(ret != AV_ERR_OK, "OH_AudioCodec_FreeOutputBuffer err: %{public}d", ret); + + ReduceWaitEncodeCnt(); +} +} // namespace Pipeline +} // namespace DistributedHardware +} // namespace OHOS diff --git a/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.h b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.h new file mode 100644 index 00000000..44e1777a --- /dev/null +++ b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_audio_encoder_filter.h @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_PIPELINE_ENCODER_FILTER_H +#define MEDIA_PIPELINE_ENCODER_FILTER_H + +#include +#include +#include +#include +#include + +#include "pipeline_status.h" +#include "filter.h" + +#include "avcodec_audio_encoder.h" +#include "avcodec_codec_name.h" +#include "avcodec_common.h" +#include "avcodec_errors.h" +#include "buffer/avbuffer_queue.h" +#include "buffer/avbuffer_queue_consumer.h" +#include "buffer/avbuffer_queue_producer.h" +#include "ffmpeg_converter.h" +#include "media_description.h" +#include "common/native_mfmagic.h" +#include "securec.h" +#include "native_avcodec_audiocodec.h" + +namespace OHOS { +namespace DistributedHardware { +namespace Pipeline { +typedef struct { + uint8_t *pcmDataBuf; + int32_t pcmLength; + int64_t timeStamp; + uint32_t streamType; + uint32_t usage; +} AEncInputData; + +typedef struct { + uint8_t *bsBuf; + int32_t bsLength; + int64_t timeStamp; + uint32_t streamType; + uint32_t usage; +} AEncOutputData; + +typedef void (*AEncOutputDataCB)(AEncOutputData *OutputDataEnc, void *privateData); +typedef struct { + int32_t codecType; + int32_t channel; + int32_t sampleRate; + MediaAVCodec::AudioSampleFormat sampleDepth; + int32_t pcmStoreType; + int32_t encoderId; + AEncOutputDataCB pfnOutput; +} AEncInitParams; + +class AudioEncoderFilter : public Filter, public std::enable_shared_from_this { +public: + AudioEncoderFilter(std::string name, FilterType type); + ~AudioEncoderFilter() override; + void Init(const std::shared_ptr& receiver, const std::shared_ptr& callback) override; + Status DoInitAfterLink() override; + Status DoPrepare() override; + Status DoStart() override; + Status DoPause() override; + Status DoPauseDragging() override; + Status DoResume() override; + Status DoResumeDragging() override; + Status DoStop() override; + Status DoFlush() override; + Status DoRelease() override; + + Status DoProcessInputBuffer(int recvArg, bool dropFrame) override; + Status DoProcessOutputBuffer(int recvArg, bool dropFrame, bool byIdx, uint32_t idx, int64_t renderTime) override; + void SetParameter(const std::shared_ptr& meta) override; + void GetParameter(std::shared_ptr& meta) override; + + Status LinkNext(const std::shared_ptr& nextFilter, StreamType outType) override; + Status UpdateNext(const std::shared_ptr& nextFilter, StreamType outType) override; + Status UnLinkNext(const std::shared_ptr& nextFilter, StreamType outType) override; + + Status OnLinked(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) override; + Status OnUpdated(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) override; + Status OnUnLinked(StreamType inType, const std::shared_ptr& callback) override; + + void OnLinkedResult(const sptr& queue, std::shared_ptr& meta); + void OnUnLinkedResult(std::shared_ptr& meta); + void OnUpdatedResult(std::shared_ptr& meta); + + Status CreateAudioCodec(); + Status ConfigureAudioCodec(const AEncInitParams &initEncParams); + Status ReleaseAudioCodec(); + Status FeedAudioData(AEncInputData *inputAudioEnc, AEncOutputData *outputAudioEnc); + + void OnEncError(int32_t errorCode); + void OnEncOutputFormatChanged(const OH_AVFormat *format); + void OnEncInputBufferAvailable(uint32_t index, OH_AVBuffer *buffer); + void OnEncOutputBufferAvailable(uint32_t index, OH_AVBuffer *buffer); + +private: + Status PrepareInputBufferQueue(); + Status SetEncoderFormat(const AEncInitParams &initEncParams); + Status CheckEncoderFormat(const AEncInitParams &initEncParams); + Status StartAudioCodec(); + Status StopAudioCodec(); + void StartInputThread(); + void StopInputThread(); + void IncreaseWaitEncodeCnt(); + void ReduceWaitEncodeCnt(); + void InputEncodeAudioData(); + Status ProcessData(std::shared_ptr audioData, const uint32_t index, OH_AVBuffer *codecMem); + int64_t GetEncoderTimeStamp(); + Status EncodeDone(AEncOutputData &outputDataEnc); +private: + constexpr static int32_t AUDIO_ENCODER_QUEUE_MAX = 100; + constexpr static int32_t ENCODE_WAIT_MILLISECONDS = 50; + constexpr static int64_t MAX_TIME_OUT_MS = 1; + constexpr static int32_t CHANNEL_MASK_MIN = 1; + constexpr static int32_t CHANNEL_MASK_MAX = 2; + constexpr static int32_t SAMPLE_RATE_MIN = 8000; + constexpr static int32_t SAMPLE_RATE_MAX = 96000; + + std::shared_ptr configureParam_ {nullptr}; + std::shared_ptr nextFilter_ {nullptr}; + std::shared_ptr eventReceiver_ {nullptr}; + std::shared_ptr filterCallback_ {nullptr}; + std::shared_ptr filterLinkCallback_ {nullptr}; + std::shared_ptr onLinkedResultCallback_ {nullptr}; + std::mutex nextFiltersMutex_; + std::shared_ptr encFliterMeta_ {nullptr}; + sptr inputProducer_ {nullptr}; + sptr outputProducer_ {nullptr}; + sptr inputConsumer_ {nullptr}; + std::shared_ptr inputBufferQueue_ {nullptr}; + static constexpr int32_t DEFAULT_BUFFER_NUM = 8; + const std::string INPUT_BUFFERQUEUE_NAME = "AvTransEncoderBufferQueue"; + static constexpr const char* ENCODE_THREAD = "encodeFilterThread"; + + std::mutex mtxData_; + std::mutex mtxCnt_; + std::thread encoderThread_; + std::condition_variable encodeCond_; + std::atomic isEncoderRunning_ = false; + int64_t firstInputTimeUs_ = 0; + int64_t inputTimeStampUs_ = 0; + int64_t outputTimeStampUs_ = 0; + int32_t waitOutputCount_ = 0; + uint32_t streamType_ = 0; + uint32_t usage_ = 0; + std::atomic isStoped_ = false; + + Media::Format cfgFormat_; + OH_AVFormat outputFormat_ = {}; + AEncInitParams initEncParams_; + OH_AVCodec *audioEncoder_ = nullptr; + std::queue codecBufQueue_; + std::queue codecIndexQueue_; + std::queue> inputDataBufferQueue_; +}; +} // namespace Pipeline +} // namespace DistributedHardware +} // namespace OHOS +#endif \ No newline at end of file diff --git a/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.cpp b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.cpp new file mode 100644 index 00000000..e6542a1d --- /dev/null +++ b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.cpp @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License") * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "av_transport_encoder_filter.h" + +#include "av_trans_log.h" +#include "filter_factory.h" + +#undef DH_LOG_TAG +#define DH_LOG_TAG "AudioEncoderFilter" + +namespace OHOS { +namespace DistributedHardware { +namespace Pipeline { +static AutoRegisterFilterg_registerAudioEncoderFilter("builtin.recorder.audioencoderfilter", + FilterType::FILTERTYPE_AENC, + [](const std::string& name, const FilterType type) { + return std::make_shared(name, FilterType::FILTERTYPE_AENC); + }); + +class AudioEncoderFilterCallback : public FilterLinkCallback { +public: + explicit AudioEncoderFilterCallback(std::shared_ptr filter) + : inputFilter_(std::move(filter)) {} + ~AudioEncoderFilterCallback() = default; + + void OnLinkedResult(const sptr& queue, + std::shared_ptr& meta) override + { + if (auto filter = inputFilter_.lock()) { + filter->OnLinkedResult(queue, meta); + } else { + AVTRANS_LOGE("Invalid inputfilter"); + } + } + void OnUnlinkedResult(std::shared_ptr& meta) override + { + if (auto filter = inputFilter_.lock()) { + filter->OnUnLinkedResult(meta); + } else { + AVTRANS_LOGE("Invalid inputfilter"); + } + } + void OnUpdatedResult(std::shared_ptr& meta) override + { + if (auto filter = inputFilter_.lock()) { + filter->OnUpdatedResult(meta); + } else { + AVTRANS_LOGE("Invalid inputfilter"); + } + } +private: + std::weak_ptr inputFilter_ {}; +}; + +class AVBufferAvaliableListener : public Media::IConsumerListener { +public: + AVBufferAvaliableListener(std::weak_ptr encoder) + { + encoder_ = encoder; + } + void OnBufferAvailable() override + { + auto encoder = encoder_.lock(); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->ProcessInputBuffer(); + } +private: + std::weak_ptr encoder_; +}; + +static void OnError(OH_AVCodec *codec, int32_t errorCode, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncError(errorCode); +} + +static void OnOutputFormatChanged(OH_AVCodec *codec, OH_AVFormat *format, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncOutputFormatChanged(format); +} + +static void OnInputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncInputBufferAvailable(index, buffer); +} + +static void OnOutputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + (void)codec; + AudioEncoderFilter *encoder = static_cast(userData); + TRUE_RETURN(encoder == nullptr, "encoder is nullptr"); + encoder->OnEncOutputBufferAvailable(index, buffer); +} + +AudioEncoderFilter::AudioEncoderFilter(std::string name, FilterType type) : Filter(name, type) +{ +} + +AudioEncoderFilter::~AudioEncoderFilter() +{ + AVTRANS_LOGI("enter"); + if (audioEncoder_ != nullptr) { + AVTRANS_LOGI("release audio encoder"); + ReleaseAudioCodec(); + } +} + +void AudioEncoderFilter::Init(const std::shared_ptr& receiver, const std::shared_ptr& callback) +{ + receiver_ = receiver; + callback_ = callback; +} + +Status AudioEncoderFilter::DoInitAfterLink() +{ + return Status::OK; +} + +Status AudioEncoderFilter::PrepareInputBufferQueue() +{ + AVTRANS_LOGI("enter"); + if (inputBufferQueue_ == nullptr) { + inputBufferQueue_ = Media::AVBufferQueue::Create(DEFAULT_BUFFER_NUM, Media::MemoryType::VIRTUAL_MEMORY, + INPUT_BUFFERQUEUE_NAME); + } + TRUE_RETURN_V_MSG_E(inputBufferQueue_ == nullptr, Status::ERROR_NULL_POINTER, "create bufferqueue failed"); + inputProducer_ = inputBufferQueue_->GetProducer(); + TRUE_RETURN_V_MSG_E(inputProducer_ == nullptr, Status::ERROR_NULL_POINTER, "GetProducer failed"); + inputConsumer_ = inputBufferQueue_->GetConsumer(); + TRUE_RETURN_V_MSG_E(inputConsumer_ == nullptr, Status::ERROR_NULL_POINTER, "GetConsumer failed"); + + sptr listener(new AVBufferAvaliableListener(shared_from_this())); + inputConsumer_->SetBufferAvailableListener(listener); + return Status::OK; +} + +Status AudioEncoderFilter::DoPrepare() +{ + AVTRANS_LOGI("enter"); + auto ret = PrepareInputBufferQueue(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "PrepareInputBufferQueue failed"); + + ret = CreateAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "Create AudioCodec failed"); + ret = ConfigureAudioCodec(initEncParams_); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "Configure AudioCodec failed"); + + TRUE_RETURN_V_MSG_E(callback_ == nullptr, Status::ERROR_NULL_POINTER, "callback is nullptr"); + callback_->OnCallback(shared_from_this(), FilterCallBackCommand::NEXT_FILTER_NEEDED, + StreamType::STREAMTYPE_RAW_AUDIO); + return Status::OK; +} + +Status AudioEncoderFilter::DoStart() +{ + AVTRANS_LOGI("enter"); + auto ret = StartAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "StartAudioCodec failed"); + return Status::OK; +} + +Status AudioEncoderFilter::DoPause() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoPauseDragging() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoResume() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoResumeDragging() +{ + return Status::OK; +} + +Status AudioEncoderFilter::DoStop() +{ + AVTRANS_LOGI("enter"); + auto ret = StopAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "StopAudioCodec failed"); + return Status::OK; +} + +Status AudioEncoderFilter::DoFlush() +{ + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + auto res = OH_AudioCodec_Flush(audioEncoder_); + TRUE_RETURN_V_MSG_E(res != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "audioEncoder flush failed: %{public}d", res); + return Status::OK; +} + +Status AudioEncoderFilter::DoRelease() +{ + AVTRANS_LOGI("enter"); + auto ret = ReleaseAudioCodec(); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "StartAudioCodec failed"); + return Status::OK; +} + +Status AudioEncoderFilter::DoProcessInputBuffer(int recvArg, bool dropFrame) +{ + AVTRANS_LOGD("enter"); + std::shared_ptr buffer = nullptr; + TRUE_RETURN_V_MSG_E(inputConsumer_ == nullptr, Status::ERROR_NULL_POINTER, "inputConsumer is null"); + Media::Status ret = inputConsumer_->AcquireBuffer(buffer); + TRUE_RETURN_V_MSG_E(ret != Media::Status::OK, Status::ERROR_INVALID_OPERATION, "AcquireBuffer failed"); + { + std::lock_guard datalock(mtxData_); + while (inputDataBufferQueue_.size() > AUDIO_ENCODER_QUEUE_MAX) { + AVTRANS_LOGE("inputDataBufferQueue_ overflow"); + auto frontBuffer = inputDataBufferQueue_.front(); + inputDataBufferQueue_.pop(); + if (frontBuffer != nullptr) { + inputConsumer_->ReleaseBuffer(frontBuffer); + } + } + inputDataBufferQueue_.push(buffer); + } + encodeCond_.notify_all(); + return Status::OK; +} + +Status AudioEncoderFilter::DoProcessOutputBuffer(int recvArg, bool dropFrame, bool byIdx, uint32_t idx, int64_t renderTime) +{ + return Status::OK; +} + +void AudioEncoderFilter::SetParameter(const std::shared_ptr& meta) +{ + encFliterMeta_ = meta; +} + +void AudioEncoderFilter::GetParameter(std::shared_ptr& meta) +{ + meta = encFliterMeta_; +} + +Status AudioEncoderFilter::LinkNext(const std::shared_ptr& nextFilter, StreamType outType) +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(nextFilter == nullptr, Status::ERROR_NULL_POINTER, "input nextFilter is nullptr"); + nextFilter_ = nextFilter; + { + std::lock_guard lock(nextFiltersMutex_); + nextFiltersMap_[outType].push_back(nextFilter); + } + auto filterLinkCallback = std::make_shared(shared_from_this()); + auto ret = nextFilter->OnLinked(outType, configureParam_, filterLinkCallback); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "Onlinked failed: %{public}d", ret); + return Status::OK; +} + +Status AudioEncoderFilter::UpdateNext(const std::shared_ptr& nextFilter, StreamType outType) +{ + return Status::OK; +} + +Status AudioEncoderFilter::UnLinkNext(const std::shared_ptr& nextFilter, StreamType outType) +{ + std::lock_guard lock(nextFiltersMutex_); + nextFiltersMap_.clear(); + return Status::OK; +} + +Status AudioEncoderFilter::OnLinked(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(callback == nullptr, Status::ERROR_NULL_POINTER, "input callback is nullptr"); + onLinkedResultCallback_ = callback; + TRUE_RETURN_V_MSG_E(meta == nullptr, Status::ERROR_NULL_POINTER, "input meta is nullptr"); + meta->GetData(Media::Tag::AUDIO_CHANNEL_COUNT, initEncParams_.channel); + meta->GetData(Media::Tag::AUDIO_SAMPLE_RATE, initEncParams_.sampleRate); + meta->GetData(Media::Tag::AUDIO_SAMPLE_FORMAT, initEncParams_.sampleDepth); // format convert needed ? + auto ret = CheckEncoderFormat(initEncParams_); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "input meta is invalid"); + return Status::OK; +} + +Status AudioEncoderFilter::OnUpdated(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) +{ + return Status::OK; +} + +Status AudioEncoderFilter::OnUnLinked(StreamType inType, const std::shared_ptr& callback) +{ + onLinkedResultCallback_ = nullptr; + return Status::OK; +} + +void AudioEncoderFilter::OnLinkedResult(const sptr& queue, + std::shared_ptr& meta) +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN(queue == nullptr, "input queue is nullptr"); + outputProducer_ = queue; + TRUE_RETURN(onLinkedResultCallback_ == nullptr, "onLinkedResultCallback_ is nullptr"); + onLinkedResultCallback_->OnLinkedResult(inputProducer_, meta); +} + +void AudioEncoderFilter::OnUnLinkedResult(std::shared_ptr& meta) +{ + +} + +void AudioEncoderFilter::OnUpdatedResult(std::shared_ptr& meta) +{ + +} + +Status AudioEncoderFilter::CreateAudioCodec() +{ + AVTRANS_LOGI("enter"); + audioEncoder_ = OH_AudioCodec_CreateByName((MediaAVCodec::AVCodecCodecName::AUDIO_ENCODER_AAC_NAME).data()); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "Create AudioCodec failed"); + OH_AVCodecCallback cb = {&OnError, &OnOutputFormatChanged, &OnInputBufferAvailable, &OnOutputBufferAvailable}; + int32_t ret = OH_AudioCodec_RegisterCallback(audioEncoder_, cb, this); + TRUE_RETURN_V_MSG_E(ret != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, "Codec setCallback failed"); + return Status::OK; +} + +Status AudioEncoderFilter::ConfigureAudioCodec(const AEncInitParams &initEncParams) +{ + AVTRANS_LOGI("enter"); + auto ret = SetEncoderFormat(initEncParams); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "input meta is invalid"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + auto res = OH_AudioCodec_Prepare(audioEncoder_); + TRUE_RETURN_V_MSG_E(res != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "audioEncoder prepare failed: %{public}d", res); + return Status::OK; +} + +Status AudioEncoderFilter::ReleaseAudioCodec() +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + bool isSuccess = true; + if (!isStoped_.load()) { + auto ret = StopAudioCodec(); + if (ret != Status::OK) { + AVTRANS_LOGE("Stop before Release AudioCodec failed"); + isSuccess = false; + } + } + auto res = OH_AudioCodec_Destroy(audioEncoder_); + if (res != AV_ERR_OK) { + AVTRANS_LOGE("Release AudioCodec failed"); + isSuccess = false; + } + audioEncoder_ = nullptr; + return isSuccess ? Status::OK : Status::ERROR_INVALID_OPERATION; +} + +Status AudioEncoderFilter::SetEncoderFormat(const AEncInitParams &initEncParams) +{ + AVTRANS_LOGI("enter"); + auto ret = CheckEncoderFormat(initEncParams); + TRUE_RETURN_V_MSG_E(ret != Status::OK, ret, "input meta is invalid"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + AVTRANS_LOGI("initEncParams channel: %{public}d, sampleRate: %{public}d, sampleDepth: %{public}d", + initEncParams.channel, initEncParams.sampleRate, initEncParams.sampleDepth); + OH_AVFormat *format = OH_AVFormat_Create(); + OH_AVFormat_SetIntValue(format, MediaAVCodec::MediaDescriptionKey::MD_KEY_CHANNEL_COUNT.data(), + initEncParams.channel); + OH_AVFormat_SetIntValue(format, MediaAVCodec::MediaDescriptionKey::MD_KEY_SAMPLE_RATE.data(), + initEncParams.sampleRate); + OH_AVFormat_SetIntValue(format, MediaAVCodec::MediaDescriptionKey::MD_KEY_AUDIO_SAMPLE_FORMAT.data(), + initEncParams.sampleDepth); + int32_t res = OH_AudioCodec_Configure(audioEncoder_, format); + if (res != AV_ERR_OK) { + AVTRANS_LOGE("configure encoder failed: %{public}d", ret); + OH_AVFormat_Destroy(format); + return Status::ERROR_INVALID_OPERATION; + } + OH_AVFormat_Destroy(format); + return Status::OK; +} + +Status AudioEncoderFilter::CheckEncoderFormat(const AEncInitParams &initEncParams) +{ + AVTRANS_LOGI("enter"); + if (initEncParams.channel >= CHANNEL_MASK_MIN && initEncParams.channel <= CHANNEL_MASK_MAX && + initEncParams.sampleRate >= SAMPLE_RATE_MIN && initEncParams.sampleRate <= SAMPLE_RATE_MAX && + initEncParams.sampleDepth == MediaAVCodec::AudioSampleFormat::SAMPLE_S16LE) { + return Status::OK; + } + AVTRANS_LOGE("initEncParams from meta err, channel: %{public}d, sampleRate: %{public}d, sampleDepth: %{public}d", + initEncParams.channel, initEncParams.sampleRate, initEncParams.sampleDepth); + return Status::ERROR_INVALID_PARAMETER; +} + +Status AudioEncoderFilter::StartAudioCodec() +{ + AVTRANS_LOGI("enter"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + auto res = OH_AudioCodec_Start(audioEncoder_); + TRUE_RETURN_V_MSG_E(res != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "audioEncoder start failed: %{public}d", res); + StartInputThread(); + return Status::OK; +} + +Status AudioEncoderFilter::StopAudioCodec() +{ + AVTRANS_LOGI("enter"); + StopInputThread(); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr, Status::ERROR_NULL_POINTER, "audioCodec is null"); + bool isSuccess = true; + auto ret = OH_AudioCodec_Flush(audioEncoder_); + if (ret != AV_ERR_OK) { + AVTRANS_LOGE("OH_AudioCodec_Flush failed"); + isSuccess = false; + } + TRUE_RETURN_V_MSG_E(OH_AudioCodec_Stop(audioEncoder_)!= AV_ERR_OK || !isSuccess, + Status::ERROR_INVALID_OPERATION, "StopAudioCodec failed"); + firstInputTimeUs_ = 0; + inputTimeStampUs_ = 0; + outputTimeStampUs_ = 0; + waitOutputCount_ = 0; + isStoped_.store(true); + return Status::OK; +} + +void AudioEncoderFilter::StartInputThread() +{ + AVTRANS_LOGI("enter"); + isEncoderRunning_.store(true); + encoderThread_ = std::thread(&AudioEncoderFilter::InputEncodeAudioData, this); + if (pthread_setname_np(encoderThread_.native_handle(), ENCODE_THREAD) != AV_ERR_OK) + { + AVTRANS_LOGE("Encode thread setname failed"); + } +} + +void AudioEncoderFilter::StopInputThread() +{ + AVTRANS_LOGI("enter"); + isEncoderRunning_.store(false); + if (encoderThread_.joinable()) { + encoderThread_.join(); + } + std::lock_guard dataLock(mtxData_); + std::queue().swap(codecIndexQueue_); + std::queue().swap(codecBufQueue_); + while (!inputDataBufferQueue_.empty()) { + auto audioData = inputDataBufferQueue_.front(); + inputDataBufferQueue_.pop(); + inputConsumer_->ReleaseBuffer(audioData); + } +} + +void AudioEncoderFilter::IncreaseWaitEncodeCnt() +{ + std::lock_guard lck(mtxCnt_); + waitOutputCount_++; +} + +void AudioEncoderFilter::ReduceWaitEncodeCnt() +{ + std::lock_guard lck(mtxCnt_); + + if (waitOutputCount_ <= 0) { + AVTRANS_LOGE("waitOutputCount_ is %{public}d", waitOutputCount_); + } + waitOutputCount_--; + AVTRANS_LOGD("waitOutputCount_ is %{public}d", waitOutputCount_); +} + +void AudioEncoderFilter::InputEncodeAudioData() +{ + AVTRANS_LOGD("enter"); + while (isEncoderRunning_.load()) { + std::shared_ptr audioData; + uint32_t index = 0; + OH_AVBuffer *codecMem; + { + std::unique_lock datalock(mtxData_); + encodeCond_.wait_for(datalock, std::chrono::milliseconds(ENCODE_WAIT_MILLISECONDS), + [this]() { + return (!inputDataBufferQueue_.empty() && !codecBufQueue_.empty()); + }); + if (inputDataBufferQueue_.empty() || codecBufQueue_.empty()) { + continue; + } + index = codecIndexQueue_.front(); + codecIndexQueue_.pop(); + codecMem = codecBufQueue_.front(); + codecBufQueue_.pop(); + audioData = inputDataBufferQueue_.front(); + inputDataBufferQueue_.pop(); + } + auto ret = ProcessData(audioData, index, codecMem); + if (ret == Status::ERROR_INVALID_OPERATION) { + AVTRANS_LOGE("Encoder is not runnnig"); + return; + } else if (ret != Status::OK) { + continue; + } + TRUE_RETURN(inputConsumer_ == nullptr, "inputConsumer is null"); + inputConsumer_->ReleaseBuffer(audioData); + } +} + +Status AudioEncoderFilter::ProcessData(std::shared_ptr audioData, const uint32_t index, + OH_AVBuffer *codecMem) +{ + AVTRANS_LOGD("enter"); + TRUE_RETURN_V_MSG_E(audioEncoder_ == nullptr || !isEncoderRunning_.load(), Status::ERROR_INVALID_OPERATION, + "Encoder is not runnnig, isEncoderRunning_: %{public}d", isEncoderRunning_.load()); + TRUE_RETURN_V_MSG_E(codecMem == nullptr || codecMem->buffer_ == nullptr || codecMem->buffer_->memory_ == nullptr, + Status::ERROR_NULL_POINTER, "codecMem is invaild"); + TRUE_RETURN_V_MSG_E(audioData == nullptr || audioData->memory_ == nullptr, + Status::ERROR_NULL_POINTER, "audioData is invaild"); + + auto memSize = audioData->memory_->GetSize(); + codecMem->buffer_->memory_->SetSize(memSize); + errno_t err = memcpy_s(OH_AVBuffer_GetAddr(codecMem), memSize, audioData->memory_->GetAddr(), memSize); + TRUE_RETURN_V_MSG_E(err != EOK, Status::ERROR_INVALID_OPERATION, + "memcpy_s err: %{public}d, memSize: %{public}d", err, memSize); + + inputTimeStampUs_ = GetEncoderTimeStamp(); + codecMem->buffer_->pts_ = inputTimeStampUs_; + codecMem->buffer_->flag_ = MediaAVCodec::AVCODEC_BUFFER_FLAG_NONE; + auto ret = OH_AudioCodec_PushInputBuffer(audioEncoder_, index); + TRUE_RETURN_V_MSG_E(ret != AV_ERR_OK, Status::ERROR_INVALID_OPERATION, + "OH_AudioCodec_PushInputBuffer err: %{public}d", ret); + + IncreaseWaitEncodeCnt(); + return Status::OK; +} + +int64_t AudioEncoderFilter::GetEncoderTimeStamp() +{ + int64_t timeIntervalStampUs = 0; + int64_t nowTimeUs = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + if (firstInputTimeUs_ == 0) { + firstInputTimeUs_ = nowTimeUs; + return timeIntervalStampUs; + } + timeIntervalStampUs = nowTimeUs - firstInputTimeUs_; + return timeIntervalStampUs; +} + +Status AudioEncoderFilter::EncodeDone(AEncOutputData &outputDataEnc) +{ + return Status::OK; +} + +void AudioEncoderFilter::OnEncError(int32_t errorCode) +{ + AVTRANS_LOGE("Encoder err: %{public}d", errorCode); + isEncoderRunning_.store(false); +} + +void AudioEncoderFilter::OnEncOutputFormatChanged(const OH_AVFormat *format) +{ + TRUE_RETURN(format == nullptr, "input format is nullptr"); + outputFormat_.format_ = format->format_; +} + +void AudioEncoderFilter::OnEncInputBufferAvailable(uint32_t index, OH_AVBuffer *buffer) +{ + AVTRANS_LOGD("enter"); + TRUE_RETURN(buffer == nullptr, "input buffer is nullptr"); + { + std::lock_guard datalock(mtxData_); + while (codecBufQueue_.size() > AUDIO_ENCODER_QUEUE_MAX || codecIndexQueue_.size() > AUDIO_ENCODER_QUEUE_MAX) { + AVTRANS_LOGE("codecBufQueue_ or codecIndexQueue_ overflow"); + codecIndexQueue_.pop(); + codecBufQueue_.pop(); + } + codecIndexQueue_.push(index); + codecBufQueue_.push(buffer); + } + encodeCond_.notify_all(); +} + +void AudioEncoderFilter::OnEncOutputBufferAvailable(uint32_t index, OH_AVBuffer *buffer) +{ + AVTRANS_LOGD("enter"); + TRUE_RETURN(audioEncoder_ == nullptr || !isEncoderRunning_.load(), + "Encoder is not runnnig, isEncoderRunning_: %{public}d", isEncoderRunning_.load()); + TRUE_RETURN(buffer == nullptr || buffer->buffer_ == nullptr || buffer->buffer_->memory_ == nullptr || + buffer->buffer_->memory_->GetSize() <= 0, "audioData is invaild"); + TRUE_RETURN(outputProducer_ == nullptr, "input queue is nullptr"); + + Media::AVBufferConfig config; + config.size = buffer->buffer_->memory_->GetSize(); + config.memoryType = Media::MemoryType::VIRTUAL_MEMORY; + config.memoryFlag = Media::MemoryFlag::MEMORY_READ_WRITE; + std::shared_ptr outBuffer = nullptr; + outputProducer_->RequestBuffer(outBuffer, config, MAX_TIME_OUT_MS); + TRUE_RETURN(outBuffer == nullptr || outBuffer->memory_ == nullptr, "RequestBuffer failed"); + auto meta = outBuffer->meta_; + if (meta == nullptr) { + AVTRANS_LOGE("outBuffer->meta_ is null"); + outputProducer_->PushBuffer(outBuffer, true); + return; + } + outBuffer->pts_ = GetEncoderTimeStamp(); + meta->SetData(Media::Tag::USER_FRAME_PTS, outBuffer->pts_); + meta->SetData(Media::Tag::AUDIO_OBJECT_NUMBER, index); + outBuffer->memory_->Write(buffer->buffer_->memory_->GetAddr(), buffer->buffer_->memory_->GetSize(), 0); + outputProducer_->PushBuffer(outBuffer, true); + outputTimeStampUs_ = buffer->buffer_->pts_; + auto ret = OH_AudioCodec_FreeOutputBuffer(audioEncoder_, index); + TRUE_RETURN(ret != AV_ERR_OK, "OH_AudioCodec_FreeOutputBuffer err: %{public}d", ret); + + ReduceWaitEncodeCnt(); +} +} // namespace Pipeline +} // namespace DistributedHardware +} // namespace OHOS diff --git a/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.h b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.h new file mode 100644 index 00000000..d81d6392 --- /dev/null +++ b/av_transport/av_trans_engine/filters/av_transport_coder/av_trans_video_encoder_filter.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_PIPELINE_ENCODER_FILTER_H +#define MEDIA_PIPELINE_ENCODER_FILTER_H + +#include +#include +#include +#include +#include + +#include "pipeline_status.h" +#include "filter.h" + +#include "avcodec_video_encoder.h" +#include "avcodec_codec_name.h" +#include "avcodec_common.h" +#include "avcodec_errors.h" +#include "buffer/avbuffer_queue.h" +#include "buffer/avbuffer_queue_consumer.h" +#include "buffer/avbuffer_queue_producer.h" +#include "ffmpeg_converter.h" +#include "media_description.h" +#include "common/native_mfmagic.h" +#include "securec.h" +#include "native_avcodec_videocodec.h" + +namespace OHOS { +namespace DistributedHardware { +namespace Pipeline { +class VideoEncoderFilter : public Filter, public std::enable_shared_from_this { +public: + VideoEncoderFilter(std::string name, FilterType type); + ~VideoEncoderFilter() override; + void Init(const std::shared_ptr& receiver, const std::shared_ptr& callback) override; + Status DoInitAfterLink() override; + Status DoPrepare() override; + Status DoStart() override; + Status DoPause() override; + Status DoPauseDragging() override; + Status DoResume() override; + Status DoResumeDragging() override; + Status DoStop() override; + Status DoFlush() override; + Status DoRelease() override; + + Status DoProcessInputBuffer(int recvArg, bool dropFrame) override; + Status DoProcessOutputBuffer(int recvArg, bool dropFrame, bool byIdx, uint32_t idx, int64_t renderTime) override; + void SetParameter(const std::shared_ptr& meta) override; + void GetParameter(std::shared_ptr& meta) override; + + Status LinkNext(const std::shared_ptr& nextFilter, StreamType outType) override; + Status UpdateNext(const std::shared_ptr& nextFilter, StreamType outType) override; + Status UnLinkNext(const std::shared_ptr& nextFilter, StreamType outType) override; + + Status OnLinked(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) override; + Status OnUpdated(StreamType inType, const std::shared_ptr& meta, + const std::shared_ptr& callback) override; + Status OnUnLinked(StreamType inType, const std::shared_ptr& callback) override; + + void OnLinkedResult(const sptr& queue, std::shared_ptr& meta); + void OnUnLinkedResult(std::shared_ptr& meta); + void OnUpdatedResult(std::shared_ptr& meta); + + Status CreateVideoCodec(); + Status ConfigureVideoCodec(const AEncInitParams &initEncParams); + Status ReleaseVideoCodec(); + Status FeedVideoData(AEncInputData *inputVideoEnc, AEncOutputData *outputVideoEnc); + + void OnEncError(int32_t errorCode); + void OnEncOutputFormatChanged(const OH_AVFormat *format); + void OnEncInputBufferAvailable(uint32_t index, OH_AVBuffer *buffer); + void OnEncOutputBufferAvailable(uint32_t index, OH_AVBuffer *buffer); + +private: + Status PrepareInputBufferQueue(); + Status SetEncoderFormat(const AEncInitParams &initEncParams); + Status CheckEncoderFormat(const AEncInitParams &initEncParams); + Status StartVideoCodec(); + Status StopVideoCodec(); + void StartInputThread(); + void StopInputThread(); + void IncreaseWaitEncodeCnt(); + void ReduceWaitEncodeCnt(); + void InputEncodeVideoData(); + Status ProcessData(std::shared_ptr videoData, const uint32_t index, OH_AVBuffer *codecMem); + int64_t GetEncoderTimeStamp(); + Status EncodeDone(AEncOutputData &outputDataEnc); +private: + constexpr static int32_t VIDEO_ENCODER_QUEUE_MAX = 100; + constexpr static int32_t ENCODE_WAIT_MILLISECONDS = 50; + constexpr static int64_t MAX_TIME_OUT_MS = 1; + constexpr static int32_t CHANNEL_MASK_MIN = 1; + constexpr static int32_t CHANNEL_MASK_MAX = 2; + constexpr static int32_t SAMPLE_RATE_MIN = 8000; + constexpr static int32_t SAMPLE_RATE_MAX = 96000; + + std::shared_ptr configureParam_ {nullptr}; + std::shared_ptr nextFilter_ {nullptr}; + std::shared_ptr eventReceiver_ {nullptr}; + std::shared_ptr filterCallback_ {nullptr}; + std::shared_ptr filterLinkCallback_ {nullptr}; + std::shared_ptr onLinkedResultCallback_ {nullptr}; + std::mutex nextFiltersMutex_; + std::shared_ptr encFliterMeta_ {nullptr}; + sptr inputProducer_ {nullptr}; + sptr outputProducer_ {nullptr}; + sptr inputConsumer_ {nullptr}; + std::shared_ptr inputBufferQueue_ {nullptr}; + static constexpr int32_t DEFAULT_BUFFER_NUM = 8; + const std::string INPUT_BUFFERQUEUE_NAME = "AvTransEncoderBufferQueue"; + static constexpr const char* ENCODE_THREAD = "encodeFilterThread"; + + std::mutex mtxData_; + std::mutex mtxCnt_; + std::thread encoderThread_; + std::condition_variable encodeCond_; + std::atomic isEncoderRunning_ = false; + int64_t firstInputTimeUs_ = 0; + int64_t inputTimeStampUs_ = 0; + int64_t outputTimeStampUs_ = 0; + int32_t waitOutputCount_ = 0; + uint32_t streamType_ = 0; + uint32_t usage_ = 0; + std::atomic isStoped_ = false; + + Media::Format cfgFormat_; + OH_AVFormat outputFormat_ = {}; + AEncInitParams initEncParams_; + OH_AVCodec *videoEncoder_ = nullptr; + std::queue codecBufQueue_; + std::queue codecIndexQueue_; + std::queue> inputDataBufferQueue_; +}; +} // namespace Pipeline +} // namespace DistributedHardware +} // namespace OHOS +#endif \ No newline at end of file -- Gitee