diff --git a/cloudphone/src/main/cpp/CMakeLists.txt b/cloudphone/src/main/cpp/CMakeLists.txt index 3225a3b2920e0f6bd827c36ea4ce4200809e77ca..25c8b5d0440702b164e4b43167cc791c90233b59 100644 --- a/cloudphone/src/main/cpp/CMakeLists.txt +++ b/cloudphone/src/main/cpp/CMakeLists.txt @@ -37,6 +37,7 @@ add_definitions(-DSOCKET_RECONNECT=1) add_definitions(-D_LOG_TO_FILE_=0) add_definitions(-DRECONNECT) add_definitions(-DUSE_TLS=1) +add_definitions(-DMTRANS_ENABLED=1) set(CMAKE_VERBOSE_MAKEFILE ON) @@ -44,6 +45,7 @@ SET(CPP_ROOT src/main/cpp) include_directories(libs/libopus/include) include_directories(libs/openssl/include/) +include_directories(libs/mtrans/include/) include_directories(cas_common) include_directories(cas_stream) @@ -80,6 +82,12 @@ set_target_properties( PROPERTIES IMPORTED_LOCATION ${THIRD_PARTY_LIB_DIR}/../libcrypto.a) +add_library(libmtrans STATIC IMPORTED) +set_target_properties( + libmtrans + PROPERTIES IMPORTED_LOCATION + ${THIRD_PARTY_LIB_DIR}/../libmtrans.a) + add_library(libopus SHARED IMPORTED) set_target_properties(libopus PROPERTIES IMPORTED_LOCATION @@ -92,6 +100,7 @@ add_library(cloudapp SHARED ${controller_src}) target_link_libraries(cloudapp ssl crypto + libmtrans cas_service cas_stream cas_common diff --git a/cloudphone/src/main/cpp/CasController.cpp b/cloudphone/src/main/cpp/CasController.cpp index 98aae1b709b162feb41adcdc5b4baebcbfbebca3..eac2b9bb68365e30ea8c0bfb03b880a6af90acbc 100644 --- a/cloudphone/src/main/cpp/CasController.cpp +++ b/cloudphone/src/main/cpp/CasController.cpp @@ -35,6 +35,11 @@ const int FRAME_RATE_MIN = 10; const int FRAME_RATE_MAX = 60; const std::string CLIENT_TYPE = "1"; +int32_t OnRecvVideoStreamData(uint8_t* data, uint32_t length); +int32_t OnRecvAudioStreamData(uint8_t* data, uint32_t length); +int32_t OnRecvCmdData(uint8_t* data, uint32_t length); +void OnNeedKeyFrameCallback(); + CasController *CasController::g_instance = nullptr; CasController *CasController::GetInstance() { @@ -82,6 +87,7 @@ CasController::CasController() m_orientation = 0; m_frameType = FrameType::H264; m_rotationDegrees = 0; + m_mtrans = nullptr; } CasController::~CasController() @@ -106,6 +112,7 @@ CasController::~CasController() m_touch = nullptr; cmdCallBack = nullptr; m_orientation = 0; + m_mtrans = nullptr; } void CasController::SetJniConf(string key, string value) @@ -271,6 +278,13 @@ bool CasController::Stop(bool isHome) StopDecWorker(!m_retainVideoDecode); CloseDataStream(); +#if MTRANS_ENABLED + if (m_mtrans != nullptr) { + delete m_mtrans; + m_mtrans = nullptr; + } +#endif + m_isNotifyFirstFrame = false; m_rotationDegrees = 0; m_orientation = 0; @@ -410,11 +424,36 @@ bool CasController::CreateWorkers() return false; } +#if MTRANS_ENABLED + if (m_mtrans == nullptr) { + TransConfigParam param; + param.minVideoSendBitrate = 2000; + param.maxVideoSendBitrate = 2000; + m_mtrans = new (std::nothrow) NetTrans(); + m_mtrans->Init(PEER_CLIENT, m_conf.ip.c_str(), m_conf.port + 2, param, + OnRecvVideoStreamData, + OnRecvAudioStreamData, + nullptr, + OnNeedKeyFrameCallback, + nullptr); + } +#endif + return true; } bool CasController::StartWorkers() { +#if MTRANS_ENABLED + if (m_mtrans != nullptr) { + if (m_mtrans->Start() < 0) { + m_mtrans->Stop(); + m_streamBuildSender->SetNetTrans(nullptr); + } else { + m_streamBuildSender->SetNetTrans(m_mtrans); + } + } +#endif m_streamParseThread->Start(); m_heartbeatThread->Start(); m_controlThread->Start(); @@ -475,6 +514,12 @@ bool CasController::DestroyWorkers() m_casClientSocket = nullptr; } +#if MTRANS_ENABLED + if (m_mtrans != nullptr) { + m_mtrans->Stop(); + } +#endif + INFO("Succeed to destroy workers"); return true; } @@ -735,6 +780,16 @@ bool CasController::SendMotionEvent(uint16_t masterAxis, int32_t masterValue, ui int CasController::JniSendData(CasMsgType type, uint8_t *data, int length) { std::lock_guard lockGuard(this->m_lock); +#if MTRANS_ENABLED + if (m_mtrans != nullptr) { + if (type == VirtualCamera) { + return length == m_mtrans->SendVideoData(data, length, "h264"); + } else if (type == VirtualMicrophone) { + return length == m_mtrans->SendAudioData(data, length); + } + } +#endif + if (m_streamBuildSender == nullptr) { return -1; } @@ -1007,4 +1062,41 @@ bool CasController::SendStartCmd() { KEY_MAX_DISCONNECT_DURATION, m_maxDisconnectDuration } }; return SendCommand(parameters); +} + +void CasController::RecvdVideoData(uint8_t *data, int length) +{ + if (m_videoPacketStream != nullptr) { + uint8_t *videoData = new uint8_t[length]; + memcpy(videoData, data, length); + m_videoPacketStream->Handle(videoData); + } +} + +void CasController::RecvdAudioData(uint8_t *data, int length) +{ + if (m_audioPacketStream != nullptr) { + uint8_t *audioData = new uint8_t[length]; + memcpy(audioData, data, length); + m_videoPacketStream->Handle(audioData); + } +} + +int32_t OnRecvVideoStreamData(uint8_t* data, uint32_t length) +{ + CasController::GetInstance()->RecvdVideoData(data, length); + return 0; +} + +int32_t OnRecvAudioStreamData(uint8_t* data, uint32_t length) +{ + int headerLen = 36; + CasController::GetInstance()->RecvdAudioData(data + headerLen, length - headerLen); + return 0; +} + +void OnNeedKeyFrameCallback() +{ + INFO("Need key frame callback.."); + CasController::GetInstance()->NotifyCommand(CAS_REQUEST_CAMERA_KEY_FRAME, "camera need key frame"); } \ No newline at end of file diff --git a/cloudphone/src/main/cpp/CasController.h b/cloudphone/src/main/cpp/CasController.h index 6698ed334e30971f597d3343f88a6a8f216b5c62..5e8beb66a58f89a13fd3d13ecabb8141456aaef9 100644 --- a/cloudphone/src/main/cpp/CasController.h +++ b/cloudphone/src/main/cpp/CasController.h @@ -32,6 +32,7 @@ #include "CasVideoHDecodeThread.h" #include "CasStreamRecvParser.h" #include "CasStreamBuildSender.h" +#include "libs/mtrans/include/net_trans.h" class CasController : public CasControllerListener { public: @@ -78,6 +79,12 @@ public: void NotifyFirstVideoFrame(); + void NotifyCommand(int type, std::string msg); + + void RecvdVideoData(uint8_t *data, int length); + + void RecvdAudioData(uint8_t *data, int length); + private: bool Release(); @@ -115,8 +122,6 @@ private: bool ForceIFrame(); - void NotifyCommand(int type, std::string msg); - void OnCmdRecv(int code, std::string msg); bool IsValidMediaConfig(std::map mediaConfig); @@ -143,6 +148,7 @@ private: CasStreamBuildSender *m_streamBuildSender = nullptr; CasStreamRecvParser *m_streamParser = nullptr; CasVideoHDecodeThread *m_videoDecodeThread = nullptr; + NetTrans *m_mtrans = nullptr; std::map m_jniConf; enum JNIState m_state = INIT; diff --git a/cloudphone/src/main/cpp/cas_service/CasVideoHDecodeThread.cpp b/cloudphone/src/main/cpp/cas_service/CasVideoHDecodeThread.cpp index 8cc327e147f5a5bd7036f24603acd775a8c996f2..130bd448a81628e934b17806cf11b25c80ad4a59 100644 --- a/cloudphone/src/main/cpp/cas_service/CasVideoHDecodeThread.cpp +++ b/cloudphone/src/main/cpp/cas_service/CasVideoHDecodeThread.cpp @@ -128,7 +128,7 @@ void DecodeTaskEntry(CasVideoHDecodeThread *decodeThreadObj) if (!lastFramePending) { int pktNum = videoPktStream->GetNumItems(); if (pktNum == 0) { - usleep(100); + usleep(16); continue; } else { DBG("Number of packet is %d", pktNum); diff --git a/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.cpp b/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.cpp index f857d95782c46f59b02e8bc852850148b49c6504..9c1036145c4af54cc16efc5c63dbea003b688556 100644 --- a/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.cpp +++ b/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.cpp @@ -29,6 +29,11 @@ CasStreamBuildSender::~CasStreamBuildSender() this->m_socket = nullptr; } +void CasStreamBuildSender::SetNetTrans(NetTrans *netTrans) +{ + m_mtrans = netTrans; +} + int CasStreamBuildSender::SendDataToServer(CasMsgType type, const void *buf, size_t len) { if (m_socket == nullptr) { @@ -114,6 +119,13 @@ int CasStreamBuildSender::SendDataToServer(CasMsgType type, const void *buf, siz for (size_t pos = 0; pos < dataLen;) { ssize_t stat = m_socket->Send(outBuffer + pos, dataLen - pos); + +#if MTRANS_ENABLED + if (m_mtrans != nullptr && type == HeartBeat) { + m_mtrans->SendCmdData(reinterpret_cast(outBuffer + pos), dataLen - pos); + } +#endif + if (stat < 0) { ERR("Socket send failed: %s.", strerror(errno)); free(outBuffer); diff --git a/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.h b/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.h index f3a6ecfa4fb5997342b58d10233cbac06ceae9ac..5c8c40f470e75f14270045232f2f71b5095ff142 100644 --- a/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.h +++ b/cloudphone/src/main/cpp/cas_stream/CasStreamBuildSender.h @@ -17,6 +17,7 @@ #include "CasMsg.h" #include "CasSocket.h" +#include "../libs/mtrans/include/net_trans.h" class CasStreamBuildSender { public: @@ -24,10 +25,13 @@ public: ~CasStreamBuildSender(); + void SetNetTrans(NetTrans *netTrans); + int SendDataToServer(CasMsgType type, const void *buf, size_t len); private: CasSocket *m_socket = nullptr; + NetTrans *m_mtrans = nullptr; }; #endif // CLOUDAPPSDK_CASSTREAMBUILDSENDER_H diff --git a/cloudphone/src/main/cpp/libs/Arm32/libmtrans.a b/cloudphone/src/main/cpp/libs/Arm32/libmtrans.a new file mode 100644 index 0000000000000000000000000000000000000000..5140bfd878c72f956a55cbf4d5fd38a447277698 Binary files /dev/null and b/cloudphone/src/main/cpp/libs/Arm32/libmtrans.a differ diff --git a/cloudphone/src/main/cpp/libs/Arm64/libmtrans.a b/cloudphone/src/main/cpp/libs/Arm64/libmtrans.a new file mode 100644 index 0000000000000000000000000000000000000000..4410512a4914f657f2fa4035c68fc872cd122a3f Binary files /dev/null and b/cloudphone/src/main/cpp/libs/Arm64/libmtrans.a differ diff --git a/cloudphone/src/main/cpp/libs/mtrans/include/net_trans.h b/cloudphone/src/main/cpp/libs/mtrans/include/net_trans.h new file mode 100644 index 0000000000000000000000000000000000000000..5b3c0b6bf20b2accafc6a5f0e4d4c29d637e4555 --- /dev/null +++ b/cloudphone/src/main/cpp/libs/mtrans/include/net_trans.h @@ -0,0 +1,49 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HUAWEICLOUD_CLOUDPHONEACCESS_ENGINE_NET_TRANS_H +#define HUAWEICLOUD_CLOUDPHONEACCESS_ENGINE_NET_TRANS_H + +#include +#include +#include "net_trans_def.h" + +class NetTrans { +public: + NetTrans(); + + virtual ~NetTrans(); + + int Init(NET_TRANS_PEER_TYPE peerType, + char const *ip, int port, + TransConfigParam param, + RecvVideoDataCallback videoCallback, + RecvAudioDataCallback audioCallback, + UpdataBitrateCallback bitrateCallback, + RequestKeyFrameCallback keyFrameCallback, + RecvCmdDataCallback cmdCallback); + + int Start(); + int Stop(); + + int SendVideoData(uint8_t *data, int len, std::string encodeType); + int SendAudioData(uint8_t *data, int len); + int SendCmdData(uint8_t *data, int len); + +private: + class NetTransPri; + class NetTransPri *m_netTransPri; +}; + +#endif //HUAWEICLOUD_CLOUDPHONEACCESS_ENGINE_NET_TRANS_H diff --git a/cloudphone/src/main/cpp/libs/mtrans/include/net_trans_def.h b/cloudphone/src/main/cpp/libs/mtrans/include/net_trans_def.h new file mode 100644 index 0000000000000000000000000000000000000000..dc8903ba5ef9a5f0fe6a75e2bb9debc3949a45e0 --- /dev/null +++ b/cloudphone/src/main/cpp/libs/mtrans/include/net_trans_def.h @@ -0,0 +1,36 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HUAWEICLOUD_CLOUDPHONEACCESS_ENGINE_NET_TRANS_DEF_H +#define HUAWEICLOUD_CLOUDPHONEACCESS_ENGINE_NET_TRANS_DEF_H + +enum NET_TRANS_PEER_TYPE { + PEER_CLIENT, + PEER_SERVER +}; + +struct TransConfigParam { + uint32_t curBandwidth; //单位 kbps + uint32_t maxBandwidth; + uint32_t minVideoSendBitrate; + uint32_t maxVideoSendBitrate; +}; + +typedef int32_t (*RecvVideoDataCallback)(uint8_t* data, uint32_t length); +typedef int32_t (*RecvAudioDataCallback)(uint8_t* data, uint32_t length); +typedef int32_t (*RecvCmdDataCallback)(uint8_t* data, uint32_t length); +typedef void (*UpdataBitrateCallback)(uint32_t encBitrate, uint32_t totalBitrate); +typedef void (*RequestKeyFrameCallback)(); + +#endif //HUAWEICLOUD_CLOUDPHONEACCESS_ENGINE_NET_TRANS_DEF_H