diff --git a/vibration_convert/core/algorithm/frequency_estimation/include/frequency_estimation.h b/vibration_convert/core/algorithm/frequency_estimation/include/frequency_estimation.h new file mode 100644 index 0000000000000000000000000000000000000000..8ba3e5f48ca960325ccb31a4668a450c7d78db27 --- /dev/null +++ b/vibration_convert/core/algorithm/frequency_estimation/include/frequency_estimation.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FREQUENCY_ESTIMATION_H +#define FREQUENCY_ESTIMATION_H + +#include +#include +#include +#include + +#include "utils.h" + +namespace OHOS { +namespace Sensors { +/** +Conversion functions +*/ +class FrequencyEstimation { +public: + FrequencyEstimation() = default; + ~FrequencyEstimation() = default; + + /** + * @brief Get the zero crossing rate + * + * @param data Audio data. + * @param frmLength The length of a frame, usually to the nth power of 2. + * @param hopLength The length of the window to be processed in a frame, usually to the nth power of 2. + * + * @return std::vector Zero crossing rate'vector. + */ + std::vector GetZeroCrossingRate(const std::vector &data, int32_t frmLength, int32_t hopLength); + + /** + * @brief Invalid frequency processing + * The default value of freqReaderAbsFlag is true. + * + * @param frequencyHz The calculated frequency includes invalid frequencies. + * @param voiceSegmentFlag Set of voice and silent segment flags. + * @param rmseIntensityNorm Normalized intensity values. + * @param freqNorm Normalized frequency values. + */ + void FreqPostProcess(const std::vector &frequencyHz, const std::vector &voiceSegmentFlag, + const std::vector &rmseIntensityNorm, std::vector &freqNorm); + +private: + double Mean(const std::vector &data); + +private: + bool freqReaderAbsFlag_ { true }; +}; +} // namespace Sensors +} // namespace OHOS +#endif // FREQUENCY_ESTIMATION_H \ No newline at end of file diff --git a/vibration_convert/core/algorithm/frequency_estimation/src/frequency_estimation.cpp b/vibration_convert/core/algorithm/frequency_estimation/src/frequency_estimation.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e2020d0033b3865cf38a8191f8834df45e8dff1b --- /dev/null +++ b/vibration_convert/core/algorithm/frequency_estimation/src/frequency_estimation.cpp @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "frequency_estimation.h" + +#include +#include +#include + +#include "sensor_log.h" +#include "sensors_errors.h" + +namespace OHOS { +namespace Sensors { +namespace { +constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, SENSOR_LOG_DOMAIN, "FrequencyEstimation" }; +constexpr double BASE_SEMITONE { 69.0 }; +constexpr double PITCH_INTERVAL_MIN { 12.0 }; +constexpr double LA_FREQUENCE { 440.0 }; +constexpr double PITCH_INTERVAL_MAX { 108.0 }; +} // namespace + +double FrequencyEstimation::Mean(const std::vector &data) +{ + if (data.empty()) { + SEN_HILOGE("data is empty"); + return 0.0; + } + double sumValue = accumulate(data.begin(), data.end(), 0.0); + return (sumValue / data.size()); +} + +void FrequencyEstimation::FreqPostProcess(const std::vector &frequencyHz, const std::vector &voiceSegmentFlag, + const std::vector &rmseIntensityNorm, std::vector &freqNorm) +{ + // Processing of effective values for filling in mute positions. + std::vector hzTrims; + int32_t ret = ProcessSilence(frequencyHz, voiceSegmentFlag, rmseIntensityNorm, hzTrims); + if (ret != Sensors::SUCCESS) { + SEN_HILOGE("ProcessSilence failed"); + return; + } + // Frequency conversion. + std::vector pitch(hzTrims.size(), 0.0); + for (size_t i = 0; i < hzTrims.size(); ++i) { + pitch[i] = BASE_SEMITONE + (PITCH_INTERVAL_MIN * log2(hzTrims[i] / LA_FREQUENCE)); + } + if (freqReaderAbsFlag_) { + freqNorm = OHOS::Sensors::NormalizePercentageRange(pitch, PITCH_INTERVAL_MIN, + (PITCH_INTERVAL_MIN + PITCH_INTERVAL_MAX)); + } else { + freqNorm = OHOS::Sensors::NormalizePercentageMin(pitch); + } + if (!freqNorm.empty() && (freqNorm.size() + 1) == rmseIntensityNorm.size()) { + freqNorm.push_back(freqNorm.back()); + } +} + +std::vector FrequencyEstimation::GetZeroCrossingRate(const std::vector &data, int32_t frmLength, int32_t hopLength) +{ + if (data.empty() || frmLength <= hopLength) { + SEN_HILOGE("data is empty or frmLength is less than hopLength"); + return {}; + } + size_t dataSize = data.size(); + size_t step = static_cast(frmLength - hopLength); + size_t frameNum = ceil(dataSize / step); + std::vector zcr(frameNum, 0.0); + std::vector oneFrmData; + auto it = data.begin(); + for (size_t i = 0; i < frameNum; ++i) { + int32_t beginIndex = i * step; + int32_t endIndex = std::min(beginIndex + frmLength, static_cast(dataSize)); + oneFrmData.assign(it + beginIndex, it + endIndex); + double curFrameMean = Mean(oneFrmData); + int32_t crossingN = 0; + size_t curFrameSize = oneFrmData.size(); + if (curFrameSize == 0) { + SEN_HILOGE("oneFrmData is empty"); + return {}; + } + for (size_t j = 0; j < (curFrameSize - 1); ++j) { + oneFrmData[j] = oneFrmData[j] - curFrameMean; + if (IsLessOrEqual(oneFrmData[j] * oneFrmData[j + 1], 0.0)) { + ++crossingN; + } + } + zcr[i] = (static_cast(crossingN) / curFrameSize); + } + return zcr; +} +} // namespace Sensors +} // namespace OHOS \ No newline at end of file diff --git a/vibration_convert/core/native/include/audio_parsing.h b/vibration_convert/core/native/include/audio_parsing.h new file mode 100644 index 0000000000000000000000000000000000000000..d8a4f011661afde16f1180d2cc99aec51b606218 --- /dev/null +++ b/vibration_convert/core/native/include/audio_parsing.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AUDIO_PARSING_H +#define AUDIO_PARSING_H + +#include +#include +#include + +#include +#include + +#include "singleton.h" +#include "vibration_convert_type.h" + +namespace OHOS { +namespace Sensors { +class AudioParsing { +public: + explicit AudioParsing(const RawFileDescriptor &fd); + ~AudioParsing() = default; + int32_t GetAudioAttribute(AudioAttribute &audioAttribute) const; + int32_t GetAudioData(int32_t samplingInterval, AudioData &audioData) const; + int32_t ConvertAudioToHaptic(const AudioSetting &audioSetting, std::vector &hapticEvents); + int32_t ParseAudioFile(); + +private: + int32_t RawFileDescriptorCheck(); + void PrintAttributeChunk(); + +private: + RawFileDescriptor rawFd_; + AudioData audioData_; + AttributeChunk attributeChunk_; +}; +} // namespace Sensors +} // namespace OHOS +#endif // AUDIO_PARSING_H \ No newline at end of file diff --git a/vibration_convert/core/native/src/audio_parsing.cpp b/vibration_convert/core/native/src/audio_parsing.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1dc57ff03a1e98a269ba39e587594a98239e8015 --- /dev/null +++ b/vibration_convert/core/native/src/audio_parsing.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "audio_parsing.h" + +#include +#include +#include +#include + +#include + +#include + +#include "generate_vibration_json_file.h" +#include "sensor_log.h" +#include "sensors_errors.h" +#include "vibration_convert_core.h" +#include "vibration_convert_type.h" + +namespace OHOS { +namespace Sensors { +namespace { +constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, Sensors::SENSOR_LOG_DOMAIN, "AudioParsing"}; +constexpr int32_t MIN_SAMPLE_COUNT = 4096; +constexpr uint32_t AUDIO_DATA_CONVERSION_FACTOR = INT32_MAX; +constexpr int32_t AUDIO_DATA_MAX_NUMBER = 100000; +constexpr int64_t LSEEK_FAIL = -1; +constexpr int32_t TIME_MS = 1000; +constexpr int32_t BITS_PER_BYTE = 8; +} // namespace + +AudioParsing::AudioParsing(const RawFileDescriptor &rawFd) +{ + CALL_LOG_ENTER; + SEN_HILOGD("handle:%{public}d, offset:%{public}" PRId64 ", length:%{public}" PRId64, + rawFd.fd, rawFd.offset, rawFd.length); + rawFd_ = rawFd; +} + +int32_t AudioParsing::RawFileDescriptorCheck() +{ + CALL_LOG_ENTER; + if ((rawFd_.fd < 0) || (rawFd_.offset < 0) || (rawFd_.length <= 0)) { + SEN_HILOGE("Invalid parameter"); + return Sensors::PARAMETER_ERROR; + } + struct stat64 statbuf = { 0 }; + if (fstat64(rawFd_.fd, &statbuf) != 0) { + SEN_HILOGE("fstat error, errno:%{public}d", errno); + return Sensors::ERROR; + } + int64_t fileSize = static_cast(statbuf.st_size); + if ((fileSize - rawFd_.offset) < rawFd_.length) { + SEN_HILOGE("Invalid parameter rawFd"); + return Sensors::PARAMETER_ERROR; + } + return Sensors::SUCCESS; +} + +int32_t AudioParsing::ParseAudioFile() +{ + CALL_LOG_ENTER; + if (RawFileDescriptorCheck() != Sensors::SUCCESS) { + SEN_HILOGE("RawFileDescriptorCheck failed"); + return Sensors::ERROR; + } + int64_t oft = lseek(rawFd_.fd, rawFd_.offset, SEEK_SET); + if (oft == LSEEK_FAIL) { + SEN_HILOGE("lseek fail, oft:%{public}" PRId64, oft); + return Sensors::ERROR; + } + (void)memset_s(&attributeChunk_, sizeof(AttributeChunk), 0, sizeof(AttributeChunk)); + if (rawFd_.length < sizeof(AttributeChunk)) { + SEN_HILOGE("Invalid parameter"); + return Sensors::PARAMETER_ERROR; + } + + ssize_t ret = read(rawFd_.fd, &attributeChunk_, sizeof(AttributeChunk)); + if (ret <= 0) { + SEN_HILOGE("read audio attribute failed, errno:%{public}d", errno); + return Sensors::ERROR; + } + if ((attributeChunk_.bitsPerSample == 0) || (attributeChunk_.fmtChannels == 0)) { + SEN_HILOGE("The divisor cannot be 0"); + return Sensors::ERROR; + } + size_t dataCount = attributeChunk_.dataSize / + ((attributeChunk_.bitsPerSample / BITS_PER_BYTE) * attributeChunk_.fmtChannels); + PrintAttributeChunk(); + int32_t *dataBuffer = static_cast(malloc(attributeChunk_.dataSize)); + CHKPR(dataBuffer, Sensors::ERROR); + (void)memset_s(dataBuffer, attributeChunk_.dataSize, 0, attributeChunk_.dataSize); + + if (rawFd_.length < attributeChunk_.dataSize) { + free(dataBuffer); + SEN_HILOGE("Invalid parameter"); + return Sensors::PARAMETER_ERROR; + } + + ret = read(rawFd_.fd, dataBuffer, attributeChunk_.dataSize); + if (ret <= 0) { + free(dataBuffer); + SEN_HILOGE("read audio data failed"); + return Sensors::ERROR; + } + for (size_t i = 0; i < dataCount; ++i) { + double data = static_cast(dataBuffer[i]) / AUDIO_DATA_CONVERSION_FACTOR; + audioData_.audioDatas.push_back(data); + } + audioData_.max = *std::max_element(audioData_.audioDatas.begin(), audioData_.audioDatas.end()); + audioData_.min = *std::min_element(audioData_.audioDatas.begin(), audioData_.audioDatas.end()); + free(dataBuffer); + return Sensors::SUCCESS; +} + +int32_t AudioParsing::GetAudioAttribute(AudioAttribute &audioAttribute) const +{ + CALL_LOG_ENTER; + if ((attributeChunk_.bitsPerSample == 0) || (attributeChunk_.fmtChannels == 0) || (attributeChunk_.byteRate == 0)) { + SEN_HILOGE("The divisor cannot be 0"); + return Sensors::ERROR; + } + audioAttribute.sampleRate = attributeChunk_.sampleRate; + audioAttribute.dataCount = attributeChunk_.dataSize / ((attributeChunk_.bitsPerSample / BITS_PER_BYTE) * + attributeChunk_.fmtChannels); + float dataSize = static_cast(attributeChunk_.dataSize); + audioAttribute.duration = static_cast(dataSize / attributeChunk_.byteRate * TIME_MS); + SEN_HILOGD("sampleRate:%{public}u, duration:%{public}u, dataCount:%{public}u", + audioAttribute.sampleRate, audioAttribute.duration, audioAttribute.dataCount); + return Sensors::SUCCESS; +} + +int32_t AudioParsing::GetAudioData(int32_t samplingInterval, AudioData &data) const +{ + CALL_LOG_ENTER; + if (audioData_.audioDatas.empty()) { + SEN_HILOGE("audioDatas is empty"); + return Sensors::ERROR; + } + size_t dataCount = audioData_.audioDatas.size(); + if (samplingInterval < 0) { + SEN_HILOGE("Invalid parameter"); + return Sensors::PARAMETER_ERROR; + } + if (samplingInterval == 0) { + if ((dataCount % AUDIO_DATA_MAX_NUMBER) == 0 && (dataCount >= AUDIO_DATA_MAX_NUMBER)) { + samplingInterval = dataCount / AUDIO_DATA_MAX_NUMBER; + } else { + samplingInterval = dataCount / AUDIO_DATA_MAX_NUMBER + 1; + } + } + for (size_t i = 0; i < dataCount; ++i) { + if (i % samplingInterval == 0) { + data.audioDatas.push_back(audioData_.audioDatas[i]); + } + } + data.max = audioData_.max; + data.min = audioData_.min; + SEN_HILOGD("min:%{public}lf, max:%{public}lf, audioDatas.size():%{public}zu", + data.min, data.max, data.audioDatas.size()); + return Sensors::SUCCESS; +} + +int32_t AudioParsing::ConvertAudioToHaptic(const AudioSetting &audioSetting, std::vector &hapticEvents) +{ + CALL_LOG_ENTER; + if (audioData_.audioDatas.size() < MIN_SAMPLE_COUNT) { + SEN_HILOGE("audioDatas less then MIN_SAMPLE_COUNT, audioDatas.size():%{public}zu", audioData_.audioDatas.size()); + return Sensors::ERROR; + } + VibrationConvertCore vibrationConvertCore; + if (vibrationConvertCore.ConvertAudioToHaptic(audioSetting, audioData_.audioDatas, hapticEvents) != Sensors::SUCCESS) { + SEN_HILOGE("ConvertAudioToHaptic failed"); + return Sensors::ERROR; + } + GenerateVibrationJsonFile generateJson; + generateJson.GenerateJsonFile(hapticEvents); + return Sensors::SUCCESS; +} + +void AudioParsing::PrintAttributeChunk() +{ + CALL_LOG_ENTER; + SEN_HILOGD("chunkID:%{public}.4s", attributeChunk_.chunkID); + SEN_HILOGD("chunkSize:%{public}u", attributeChunk_.chunkSize); + SEN_HILOGD("format:%{public}.4s", attributeChunk_.format); + SEN_HILOGD("fmtID:%{public}.4s", attributeChunk_.fmtID); + SEN_HILOGD("fmtSize:%{public}u", attributeChunk_.fmtSize); + SEN_HILOGD("fmtTag:%{public}hu", attributeChunk_.fmtTag); + SEN_HILOGD("fmtChannels:%{public}hu", attributeChunk_.fmtChannels); + SEN_HILOGD("sampleRate:%{public}u", attributeChunk_.sampleRate); + SEN_HILOGD("byteRate:%{public}u", attributeChunk_.byteRate); + SEN_HILOGD("blockAilgn:%{public}hu", attributeChunk_.blockAilgn); + SEN_HILOGD("bitsPerSample:%{public}hu", attributeChunk_.bitsPerSample); + SEN_HILOGD("dataID:%{public}.4s", attributeChunk_.dataID); + SEN_HILOGD("dataSize:%{public}u", attributeChunk_.dataSize); +} +} // namespace Sensors +} // namespace OHOS diff --git a/vibration_convert/interfaces/js/src/vibrator_convert_napi_utils.cpp b/vibration_convert/interfaces/js/src/vibrator_convert_napi_utils.cpp index b3107e95cf8ef74fda1bcc1493bfe73d8c3e87f2..d1021c8125c2860e890d74fb061a6bfeaebde044 100644 --- a/vibration_convert/interfaces/js/src/vibrator_convert_napi_utils.cpp +++ b/vibration_convert/interfaces/js/src/vibrator_convert_napi_utils.cpp @@ -216,7 +216,7 @@ napi_value GetAudioDataInfo(sptr asyncCallbackInfo) CHKCP((napi_create_double(env, datas.max, &max) == napi_ok), "napi_create_double max failed"); CHKCP((napi_set_named_property(env, object, "dataMax", max) == napi_ok), "napi_set_named_property dataMax failed"); napi_value data = nullptr; - if (!CreateArray(env, datas.audioData, data)) { + if (!CreateArray(env, datas.audioDatas, data)) { SEN_HILOGE("CreateArray failed"); return nullptr; }