diff --git a/cloudphone/src/main/cpp/CasController.cpp b/cloudphone/src/main/cpp/CasController.cpp index 5086f472c3631462ba8d272d7ee0f3a505444ff3..165fa8ae1fae697bcdb1c275b3f5e43cfc981208 100644 --- a/cloudphone/src/main/cpp/CasController.cpp +++ b/cloudphone/src/main/cpp/CasController.cpp @@ -26,6 +26,7 @@ #include "CasExtVideoDataPipe.h" #include "opus.h" #include "CasVideoUtil.h" +#include using namespace std; @@ -41,17 +42,30 @@ const int FRAME_RATE_MAX = 60; const std::string CLIENT_TYPE = "1"; const std::string HRTP_LOG_PATH = "/sdcard/Android/media/com.huawei.cloudapp.b004/"; const uint64_t DURATION_USEC = 1000000ULL; +uint32_t m_mrightEncBitrate = 2000000; // 初始化动态码率 +// 注册mright回调 int32_t OnRecvVideoStreamData(uint8_t* data, uint32_t length); int32_t OnRecvAudioStreamData(uint8_t* data, uint32_t length); int32_t OnRecvAudioDecodeCallback(int32_t streamId, AudioJbDecode* audioJbDecode); int32_t OnRecvCmdData(uint8_t* data, uint32_t length); void OnGotTransLog(const char* str, uint32_t length); +void OnRecvBitrate(uint32_t encBitrate, uint32_t totalBitrate); void OnNeedKeyFrameCallback(); OpusDecoder *g_opusDecoder = nullptr; shared_ptr hrtpLogger = nullptr; int g_plcCount = 0; +uint64_t GetCurrentTimeMs() +{ + constexpr int oneSecond = 1000; + timespec now; + if (clock_gettime(CLOCK_REALTIME, &now) != 0){ + ERR("get time failed"); + } + return reinterpret_cast(now.tv_sec * oneSecond + now.tv_nsec/(oneSecond*oneSecond)); +} + CasController *CasController::g_instance = nullptr; CasController *CasController::GetInstance() { @@ -440,10 +454,11 @@ bool CasController::CreateWorkers() #if MTRANS_ENABLED if (m_mtrans == nullptr) { - TransConfigParam param; - param.minVideoSendBitrate = 500; - param.maxVideoSendBitrate = 10000; - + TransConfigParam param; + param.minVideoSendBitrate = 750; + param.maxVideoSendBitrate = 5000; + param.curBandwidth = 5000; + param.maxBandwidth = 20000; if (!m_logInit) { m_logInit = true; // Create a file rotating logger with 50 MB size max and 3 rotated files @@ -454,14 +469,15 @@ bool CasController::CreateWorkers() m_mtrans = new (std::nothrow) NetTrans(); m_mtrans->EnableLog(HRTP_LOG_PATH + "hrtp_log.txt", TRANS_LOG_LEVEL_INFO); - m_mtrans->Init(PEER_CLIENT, m_conf.ip.c_str(), m_conf.port, param, + int res = m_mtrans->Init(PEER_CLIENT, m_conf.ip.c_str(), m_conf.port, param, OnRecvVideoStreamData, OnRecvAudioStreamData, - nullptr, + OnRecvBitrate, OnNeedKeyFrameCallback, OnRecvCmdData, OnRecvAudioDecodeCallback, OnGotTransLog); + INFO("m_mtrans->Init result = %d", res); } #endif @@ -763,6 +779,58 @@ bool CasController::SendTouchEvent(int id, int action, int x, int y, int pressur return false; } + #if MTRANS_ENABLED + if (m_mtrans != nullptr && m_isMTransValid) { + size_t pkgLen = sizeof(TouchEventParam); + stream_msg_head_t msgHead; + msgHead.checksum = CAS_MSG_CHECKSUM_TOUCH_INPUT; + msgHead.SetPayloadSize(pkgLen); + msgHead.magicword = CAS_STREAM_DELIMITER_MAGICWORD; + msgHead.type = TouchInput; + + TouchEventParam touchEventParam{}; + touchEventParam.id = static_cast(id); + touchEventParam.action = static_cast(action); + touchEventParam.x = (unsigned short)htons(x); + touchEventParam.y = (unsigned short)htons(y); + touchEventParam.pressure = (unsigned short)htons(pressure); + touchEventParam.time = htonl(time); + touchEventParam.orientation = static_cast(orientation); + touchEventParam.height = (unsigned short)htons(height); + touchEventParam.width = (unsigned short)htons(width); + + size_t dataLen = sizeof(stream_msg_head_t) + pkgLen; + + char *outBuffer = (char *)malloc(dataLen); + if (outBuffer == nullptr) { + ERR("Failed to malloc out buffer."); + return 0; + } + if (EOK != memcpy_s(outBuffer, dataLen, &msgHead, sizeof(stream_msg_head_t))) { + ERR("Copy msg head fail."); + free(outBuffer); + return 0; + } + if (EOK != memcpy_s(outBuffer + sizeof(stream_msg_head_t), pkgLen, &touchEventParam, pkgLen)) { + ERR("Copy msg data fail."); + free(outBuffer); + return 0; + } + + uint64_t beforeTime = GetCurrentTimeMs(); + int ret = m_mtrans->SendTouchEventData(reinterpret_cast(outBuffer), dataLen); + uint64_t afterTime = GetCurrentTimeMs(); + uint64_t gapTime = afterTime - beforeTime; + INFO("cyl The timestamp before touch is %lld, after touch is %lld, gap is %lld", beforeTime, afterTime, gapTime); + if (ret != static_cast(dataLen)) { + ERR("Failed to send touch event, aimed to send:%d, sent:%d", (int)dataLen, ret); + return false; + } + // INFO("cyl mright send touch event success."); + return true; + } +#endif + if (m_touch == nullptr) { return false; } @@ -781,6 +849,47 @@ bool CasController::SendKeyEvent(uint16_t keycode, uint16_t action) return false; } +#if MTRANS_ENABLED + if (m_mtrans != nullptr && m_isMTransValid) { + size_t pkgLen = sizeof(KeyEventParam); + stream_msg_head_t msgHead; + msgHead.checksum = CAS_MSG_CHECKSUM_KEYEVENTINPUT; + msgHead.SetPayloadSize(pkgLen); + msgHead.magicword = CAS_STREAM_DELIMITER_MAGICWORD; + msgHead.type = KeyEventInput; + + KeyEventParam keyEventParam{}; + keyEventParam.keycode = (unsigned short) htons(keycode); + keyEventParam.action = (unsigned short) htons(action); + + size_t dataLen = sizeof(stream_msg_head_t) + pkgLen; + + char *outBuffer = (char *)malloc(dataLen); + if (outBuffer == nullptr) { + ERR("Failed to malloc out buffer."); + return 0; + } + if (EOK != memcpy_s(outBuffer, dataLen, &msgHead, sizeof(stream_msg_head_t))) { + ERR("Copy msg head fail."); + free(outBuffer); + return 0; + } + if (EOK != memcpy_s(outBuffer + sizeof(stream_msg_head_t), pkgLen, &keyEventParam, pkgLen)) { + ERR("Copy msg data fail."); + free(outBuffer); + return 0; + } + + int ret = m_mtrans->SendKeyEventData(reinterpret_cast(outBuffer), dataLen); + if (ret != static_cast(dataLen)) { + ERR("Failed to send key event, aimed to send:%d, sent:%d", (int)sizeof(KeyEventParam), ret); + return false; + } + INFO("cyl mright send key event success."); + return true; + } +#endif + if (m_touch == nullptr) { return false; } @@ -859,6 +968,10 @@ int CasController::JniSendData(CasMsgType type, uint8_t *data, int length) return length == m_mtrans->SendVideoData(data, length, "h264"); case (VirtualMicrophone): return length == m_mtrans->SendAudioData(data, length); + case (VirtualSensor): + return length == m_mtrans->SendSensorData(data, length); + case (VirtualLocation): + return length == m_mtrans->SendLocationData(data, length); default: break; } @@ -1097,6 +1210,11 @@ void CasController::NotifyFirstVideoFrame() } } +void CasController::NotifyBitrate(uint32_t bitrate) +{ + bitrateCallBack(bitrate); +} + void CasController::OnCmdRecv(int code, string msg) { if (code == CAS_H265_NOT_SUPPORT) { @@ -1185,6 +1303,7 @@ int32_t OnRecvAudioStreamData(uint8_t* data, uint32_t length) msgHead.magicword = CAS_STREAM_DELIMITER_MAGICWORD; msgHead.type = Audio; msgHead.SetPayloadSize(length); + ERR("cxc OnRecvAudioStreamData is %d:", length); size_t dataLen = sizeof(stream_msg_head_t) + length; char *outBuffer = (char *)malloc(dataLen); @@ -1241,6 +1360,14 @@ void OnGotTransLog(const char* str, uint32_t length) hrtpLogger->info(str); } +void OnRecvBitrate(uint32_t encBitrate, uint32_t totalBitrate) +{ + // 动态码率 + m_mrightEncBitrate = encBitrate * 1000; // 初始化动态码率 + INFO("m_mrightEncBitrate set encBitrate is %ld", m_mrightEncBitrate); + CasController::GetInstance()->NotifyBitrate(m_mrightEncBitrate); +} + void OnNeedKeyFrameCallback() { INFO("Need key frame callback.."); diff --git a/cloudphone/src/main/cpp/CasController.h b/cloudphone/src/main/cpp/CasController.h index 9064f6fec56b4d935045417a1a3f7903fee60e3b..9b6dd9142fcc5dd70053071324b7755d488156f0 100644 --- a/cloudphone/src/main/cpp/CasController.h +++ b/cloudphone/src/main/cpp/CasController.h @@ -65,6 +65,11 @@ public: cmdCallBack = method; } + void SetBitrateCallBackMethod(void *(*method)(uint32_t bitrate)) + { + bitrateCallBack = method; + } + CasVideoHDecodeThread* GetVideoDecodeThread() { return m_videoDecodeThread; @@ -86,6 +91,8 @@ public: void NotifyCommand(int type, std::string msg); + void NotifyBitrate(uint32_t bitrate); + void RecvdVideoData(uint8_t *data, int length); void RecvdAudioData(uint8_t *data, int length); @@ -143,6 +150,7 @@ private: int GetCurrentFPS() const; void *(*cmdCallBack)(int type, std::string msg) = nullptr; + void *(*bitrateCallBack)(uint32_t bitrate) = nullptr; static CasController *g_instance; CasDataPipe *m_videoPacketStream = nullptr; diff --git a/cloudphone/src/main/cpp/CasJniBridge.cpp b/cloudphone/src/main/cpp/CasJniBridge.cpp index b4b470e2783ca5bfff157a910cfcaf84a649e448..d798ad4dbb8ab50d20f844ba6c21598bff84e066 100644 --- a/cloudphone/src/main/cpp/CasJniBridge.cpp +++ b/cloudphone/src/main/cpp/CasJniBridge.cpp @@ -34,6 +34,7 @@ CasController *gJniApiCtrl = CasController::GetInstance(); static JavaVM *g_JVM = nullptr; static jobject jniCallback = nullptr; void *invokeCmdCallBack(int type, string msg); +void *invokeMtransBitrateUpdate(uint32_t bitrate); static std::string jstring2string(JNIEnv *env, jstring jStr) { @@ -242,6 +243,41 @@ extern "C" JNIEXPORT void JNICALL JNI(registerCasJNICallback)(JNIEnv *env, jclas env->GetJavaVM(&g_JVM); jniCallback = env->NewGlobalRef(callback); gJniApiCtrl->SetCmdCallBackMethod(invokeCmdCallBack); + gJniApiCtrl->SetBitrateCallBackMethod(invokeMtransBitrateUpdate); +} + +void *invokeMtransBitrateUpdate(uint32_t bitrate) +{ + JNIEnv *env = nullptr; + int mNeedDetach = 0; + int getEnvStat = g_JVM->GetEnv((void **)&env, JNI_VERSION_1_4); + INFO("Get env stat %d and bitrate %d.", getEnvStat, bitrate); + if (getEnvStat == JNI_EDETACHED) { + if (g_JVM->AttachCurrentThread(&env, nullptr) != 0) { + ERR("Attach current thread failed."); + return nullptr; + } + mNeedDetach = JNI_TRUE; + } + jclass javaClass = env->GetObjectClass(jniCallback); + if (javaClass == 0) { + ERR("Unable to find class"); + g_JVM->DetachCurrentThread(); + return nullptr; + } + + jmethodID javaCallbackId = env->GetMethodID(javaClass, "onBitrateCallback", "(I)V"); + if (javaCallbackId == nullptr) { + ERR("Unable to find method."); + return nullptr; + } + env->CallVoidMethod(jniCallback, javaCallbackId, bitrate); + INFO("Invoke bitrate callback end."); + if (mNeedDetach) { + g_JVM->DetachCurrentThread(); + } + env = nullptr; + return nullptr; } void *invokeCmdCallBack(int type, string msg) diff --git a/cloudphone/src/main/cpp/libs/Arm32/libmtrans.a b/cloudphone/src/main/cpp/libs/Arm32/libmtrans.a index c5ea687fc5b3b4d8e3f0d0ae71b159764423fe2a..7cefc847f1f8fc9625a19023384d49b256e5779e 100644 Binary files a/cloudphone/src/main/cpp/libs/Arm32/libmtrans.a and b/cloudphone/src/main/cpp/libs/Arm32/libmtrans.a differ diff --git a/cloudphone/src/main/cpp/libs/Arm32/libmtransB004.a b/cloudphone/src/main/cpp/libs/Arm32/libmtransB004.a new file mode 100644 index 0000000000000000000000000000000000000000..c5ea687fc5b3b4d8e3f0d0ae71b159764423fe2a Binary files /dev/null and b/cloudphone/src/main/cpp/libs/Arm32/libmtransB004.a differ diff --git a/cloudphone/src/main/cpp/libs/Arm64/libmtrans.a b/cloudphone/src/main/cpp/libs/Arm64/libmtrans.a index 5cc7394353ed5939ec60b3dc2ef982209d5b98e5..b0e8295dfb10bb188598d1e88135f18c1cdf5638 100644 Binary files a/cloudphone/src/main/cpp/libs/Arm64/libmtrans.a and b/cloudphone/src/main/cpp/libs/Arm64/libmtrans.a differ diff --git a/cloudphone/src/main/cpp/libs/Arm64/libmtransB004.a b/cloudphone/src/main/cpp/libs/Arm64/libmtransB004.a new file mode 100644 index 0000000000000000000000000000000000000000..5cc7394353ed5939ec60b3dc2ef982209d5b98e5 Binary files /dev/null and b/cloudphone/src/main/cpp/libs/Arm64/libmtransB004.a differ diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java b/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java index 51a20670fa8ce6f12f4b9f03d390f4c1df78f7fc..5143df3d444088d861f8263bced5191b87b46a9a 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java @@ -60,6 +60,7 @@ import com.huawei.cloudphone.common.CasState; import com.huawei.cloudphone.jniwrapper.JNIWrapper; import com.huawei.cloudphone.service.CasProcessor; import com.huawei.cloudphone.virtualdevice.VirtualDeviceSession; +import com.huawei.cloudphone.virtualdevice.camera.VirtualCamera; import com.huawei.cloudphone.virtualdevice.camera.VirtualCameraManager; import com.huawei.cloudphone.virtualdevice.common.RingBufferVirtualDeviceIO; import com.huawei.cloudphone.virtualdevice.common.VirtualDeviceManager; @@ -940,6 +941,23 @@ public class CloudPhoneImpl implements ICloudPhone { } } + public void onBitrateRecv(int bitrate) throws RemoteException { + VirtualDeviceProtocol virtualDevice = mVirtualDeviceSession.getVirtualDevice(); + if (virtualDevice == null) { + return ; + } + VirtualDeviceManager virtualDeviceManager = virtualDevice.getVirtualDeviceManager(DEV_TYPE_CAMERA); + if (virtualDeviceManager == null) { + return ; + } + + int lastBitrate = ((VirtualCameraManager) virtualDeviceManager).getBitrate(); + // 根据m_trans回调码率，当码率变动超过一定阈值，动态更新上行数据传输码率 + if (Math.abs(bitrate - lastBitrate) >= 500000) { + CASLog.e(TAG, "Recieve last Bitrate is " + lastBitrate + ". But recieve new Bitrate is " + bitrate); + ((VirtualCameraManager) virtualDeviceManager).updateDynamicBitrate(bitrate); + } + } public void onChannelDataRecv(byte[] data) throws RemoteException { if (mChannelDataListener != null) { mChannelDataListener.onRecvCloudAppData(data); diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/service/CasInteractiveStateCallback.java b/cloudphone/src/main/java/com/huawei/cloudphone/service/CasInteractiveStateCallback.java index e8c758613d80e3c6b937c408b17268aacdf61466..b4552279d03149286c3cfca5d4db543e8e09981c 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/service/CasInteractiveStateCallback.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/service/CasInteractiveStateCallback.java @@ -24,4 +24,6 @@ public interface CasInteractiveStateCallback { * on cmd receive */ void onCmdReceive(int code, String msg); + + void onBitrateCallback(int bitrate); } diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/service/CasProcessor.java b/cloudphone/src/main/java/com/huawei/cloudphone/service/CasProcessor.java index 1967f41a4b82de4760efdfc019166531d49e8eb2..0ace727385799734aba95b5cb65025b1492c19ec 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/service/CasProcessor.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/service/CasProcessor.java @@ -99,10 +99,24 @@ public class CasProcessor { try { mListener.onCmdRecv(code, msg); } catch (RemoteException e) { - CASLog.e(TAG, "failed to notify cas state."); + CASLog.e(TAG, "onCmdReceive::failed to notify cas state."); } } } + + @Override + public void onBitrateCallback(int bitrate) { + CASLog.i(TAG, "Bitrate call back from jni m_trans send bitrate is: " + bitrate); + if (mListener != null) { + try { + mListener.onBitrateRecv(bitrate); + } catch (RemoteException e) { + CASLog.e(TAG, "onBitrateCallback::failed to notify cas state."); + } + } + + + } }; JniBridge.getInstance().registerCasJNICallback(interactiveStateCallback); } diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java index f0b3bd9acbd208d1072b5c45791e295aaa07d1f5..44e72f27840654eab749816fdeea59c30320950b 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java @@ -62,6 +62,15 @@ public class AvcEncoder { } } + // 支持MediaCodec动态设置上行传输码率 + public void updateDynamicBitrate(int newBitrate) { + if (Build.VERSION.SDK_INT >= M && mediaCodec != null) { + Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, newBitrate); + mediaCodec.setParameters(params); + } + } + public AvcEncoder(int width, int height, int frameRate, int bitrate, AvcEncoderDataHandler dataHandler) { Log.i(TAG, "Create AvcEncoder, width = " + width + ", height = " + height + ", frameRate = " + frameRate + ", bitrate = " + bitrate); @@ -96,6 +105,7 @@ public class AvcEncoder { } } + // 将输入的视频帧进行编码并输出H264格式的视频流。 public void offerEncoder(byte[] frame, byte[] encodeBuff) { if ((frame == null) || (encodeBuff == null)) { Log.e(TAG, "Offer Encoder input param invalid. "); @@ -142,6 +152,7 @@ public class AvcEncoder { } } + // 视频编码器的输入缓冲区处理函数。它将视频帧转换为YUV格式 private void inputBuffer(byte[] frame) { long pts = computePresentationTime(genIndex); genIndex++; diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java index f709b26d7bb9836dbdaa92eb1e10f381ff5022ed..f4d00a686b2c7fa36cdc60e48672a80e93ddc471 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java @@ -21,11 +21,15 @@ import static android.hardware.Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO; import static android.hardware.Camera.Parameters.FOCUS_MODE_FIXED; import static android.hardware.Camera.Parameters.FOCUS_MODE_INFINITY; import static android.hardware.Camera.Parameters.FOCUS_MODE_MACRO; +import static android.os.Build.VERSION_CODES.M; import android.graphics.ImageFormat; import android.graphics.SurfaceTexture; import android.hardware.Camera; +import android.media.MediaCodec; import android.opengl.GLES11Ext; +import android.os.Build; +import android.os.Bundle; import android.util.Log; import android.view.SurfaceHolder; @@ -52,7 +56,7 @@ public class VirtualCamera implements Camera.PreviewCallback { private int mWidth = DEFAULT_WIDTH; private int mHeight = DEFAULT_HEIGHT; - private int mFps = 15; + private int mFps = 30; private int mBitrate = 2000000; private SurfaceHolder mSurfaceHolder; private boolean mIsUseH264; @@ -182,6 +186,8 @@ public class VirtualCamera implements Camera.PreviewCallback { return cameraInfo.orientation; } + public int getBitrate() { return mBitrate; } + public byte[] getParameters() { return mParameters.flatten().getBytes(); } @@ -197,6 +203,11 @@ public class VirtualCamera implements Camera.PreviewCallback { mFps = fps; } + public void setBitrate(int bitrate) { + Log.i(TAG, "virtualcamera set bitrate is: " + bitrate); + mBitrate = bitrate; + } + public boolean isSupportH264() { return AvcEncoder.isSupportH264(); } @@ -270,6 +281,12 @@ public class VirtualCamera implements Camera.PreviewCallback { } } + public void updateDynamicBitrate(int newBitrate) { + if (mAvcCodec != null) { + mAvcCodec.updateDynamicBitrate(newBitrate); + } + } + class H264DataHandler implements AvcEncoderDataHandler { @Override public void handleData(byte[] data, int length) { diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java index e4f14552406bc3eec08bcf2eefa0a093a838b714..0984a2d48987369a76d31e4da179bbd8e756da8a 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java @@ -84,6 +84,14 @@ public class VirtualCameraManager extends VirtualDeviceManager { mPermissionListener = listener; } + public void updateDynamicBitrate(int bitrate) { + mVirtualCamera.updateDynamicBitrate(bitrate); + } + + public int getBitrate() { + return mVirtualCamera.getBitrate(); + } + public void processMsg(MsgHeader header, byte[] body) { switch (header.mOptType) { case OPT_CAMERA_GET_PARAM_REQ: