From 3bfd7cd51d26c6a4bf0a3dd5259029f572fa8ca7 Mon Sep 17 00:00:00 2001 From: wangshuo <584363327@qq.com> Date: Fri, 17 Nov 2023 11:30:09 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=91=84=E5=83=8F=E5=A4=B4=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E6=8F=90=E4=BE=9B=E5=BC=BA=E5=88=B6i=E5=B8=A7?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cloudphone/src/main/cpp/CasCommon.h | 3 ++- .../cloudphone/apiimpl/CloudPhoneImpl.java | 15 ++++++++++++- .../huawei/cloudphone/common/CasState.java | 1 + .../virtualdevice/VirtualDeviceSession.java | 11 ++++++---- .../virtualdevice/camera/AvcEncoder.java | 21 +++++++++++++------ .../virtualdevice/camera/VirtualCamera.java | 11 ++++++---- .../camera/VirtualCameraManager.java | 5 +++++ .../common/VirtualDeviceProtocol.java | 7 +++++++ 8 files changed, 58 insertions(+), 16 deletions(-) diff --git a/cloudphone/src/main/cpp/CasCommon.h b/cloudphone/src/main/cpp/CasCommon.h index 840f9b6..1e0367c 100644 --- a/cloudphone/src/main/cpp/CasCommon.h +++ b/cloudphone/src/main/cpp/CasCommon.h @@ -83,8 +83,9 @@ enum { CAS_SWITCH_FOREGROUND_ERROR = 0x1501, CAS_ORIENTATION = 0x1600, - CAS_EXIT = 0x1700, + CAS_EXIT = 0x1700, CAS_BACK_HOME = 0x1900, + CAS_REQUEST_CAMERA_KEY_FRAME = 0x2000, }; const int CAS_THREAD_RUNNING = 1; diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java b/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java index cfafdee..931afdf 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/apiimpl/CloudPhoneImpl.java @@ -51,7 +51,10 @@ import com.huawei.cloudphone.common.CasState; import com.huawei.cloudphone.jniwrapper.JNIWrapper; import com.huawei.cloudphone.service.CasProcessor; import com.huawei.cloudphone.virtualdevice.VirtualDeviceSession; +import com.huawei.cloudphone.virtualdevice.camera.VirtualCameraManager; import com.huawei.cloudphone.virtualdevice.common.RingBufferVirtualDeviceIO; +import com.huawei.cloudphone.virtualdevice.common.VirtualDeviceManager; +import com.huawei.cloudphone.virtualdevice.common.VirtualDeviceProtocol; import java.util.HashMap; import java.util.Map; @@ -61,6 +64,7 @@ import java.util.TimerTask; import static android.view.KeyEvent.KEYCODE_BACK; import static android.view.KeyEvent.KEYCODE_VOLUME_DOWN; import static android.view.KeyEvent.KEYCODE_VOLUME_UP; +import static com.huawei.cloudphone.api.CloudPhoneParas.DEV_TYPE_CAMERA; import static com.huawei.cloudphone.api.CloudPhoneParas.DisplayMode.DISPLAY_MODE_FILL; import static com.huawei.cloudphone.common.CasState.CAS_CONNECT_EXCEPTION; import static com.huawei.cloudphone.utils.CasMediaUtils.isValidMediaConfig; @@ -571,7 +575,6 @@ public class CloudPhoneImpl implements ICloudPhone { } private void handleStateChangeCmd(Message msg) { - //当cae返回如下这些状态时，sdk主动发起stop synchronized (mCloudPhoneLock) { try { int state = msg.arg1; @@ -606,6 +609,16 @@ public class CloudPhoneImpl implements ICloudPhone { if (mVirtualDeviceSession != null) { mVirtualDeviceSession.start(); } + } else if (state == CasState.CAS_REQUEST_CAMERA_KEY_FRAME) { + VirtualDeviceProtocol virtualDevice = mVirtualDeviceSession.getVirtualDevice(); + if (virtualDevice == null) { + return; + } + VirtualDeviceManager virtualDeviceManager = virtualDevice.getVirtualDeviceManager(DEV_TYPE_CAMERA); + if (virtualDeviceManager == null) { + return; + } + ((VirtualCameraManager) virtualDeviceManager).generateKeyFrame(); } } catch (Exception e) { CASLog.e(TAG, "handleStateChangeCmd failed. " + e.getMessage()); diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/common/CasState.java b/cloudphone/src/main/java/com/huawei/cloudphone/common/CasState.java index 163f2ed..6f5a6bf 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/common/CasState.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/common/CasState.java @@ -54,6 +54,7 @@ public class CasState { static public final int CAS_SWITCH_BACKGROUND_ERROR = 0x1301; static public final int CAS_SWITCH_FOREGROUND_SUCCESS = 0x1400; static public final int CAS_EXIT = 0x1700; + static public final int CAS_REQUEST_CAMERA_KEY_FRAME = 0x2000; static public final int CAS_INVALID_CMD = 0xFFFF; public static class CasStateMsg { diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/VirtualDeviceSession.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/VirtualDeviceSession.java index a36a03c..ce0a5eb 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/VirtualDeviceSession.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/VirtualDeviceSession.java @@ -36,14 +36,17 @@ public class VirtualDeviceSession { mContext = context; } - public void setCameraSurfaceHolder(SurfaceHolder surfaceHolder) { - mParamBundle.setSurfaceHolder(surfaceHolder); - } - public void setVirtualDeviceIoHook(IVirtualDeviceIO virtualDeviceIO) { mVirtualDeviceIO = virtualDeviceIO; } + public VirtualDeviceProtocol getVirtualDevice() { + if (mVirtualDevice != null) { + return mVirtualDevice; + } + return null; + } + public void start() { synchronized (mLock) { mVirtualDevice = new VirtualDeviceProtocol(mVirtualDeviceIO, mContext); diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java index f488194..b54df3c 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java @@ -17,12 +17,15 @@ package com.huawei.cloudphone.virtualdevice.camera; import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar; import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar; +import static android.os.Build.VERSION_CODES.M; import android.graphics.ImageFormat; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.MediaFormat; +import android.os.Build; +import android.os.Bundle; import android.util.Log; import java.io.IOException; @@ -44,8 +47,18 @@ public class AvcEncoder { private byte[] configByte; private AvcEncoderDataHandler dataHandler; - public void getIFrame() { - mediaCodec.flush(); + public void flush() { + if (mediaCodec != null) { + mediaCodec.flush(); + } + } + + public void generateKeyFrame() { + if (Build.VERSION.SDK_INT >= M && mediaCodec != null) { + Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); + mediaCodec.setParameters(params); + } } public AvcEncoder(int width, int height, int frameRate, int bitrate, AvcEncoderDataHandler dataHandler) { @@ -60,7 +73,6 @@ public class AvcEncoder { mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat); mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate); - mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL); mediaFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR); try { @@ -92,8 +104,6 @@ public class AvcEncoder { while (true) { MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); int outBufId = mediaCodec.dequeueOutputBuffer(bufferInfo, TIMEOUT_USC); - Log.i(TAG, "bufferInfo: size = " + bufferInfo.size + ", offset = " + bufferInfo.offset + - ", flags = " + bufferInfo.flags + ", presentationTimeUs = " + bufferInfo.presentationTimeUs); if (outBufId >= 0) { ByteBuffer outBuf = mediaCodec.getOutputBuffer(outBufId); byte[] h264Buf = new byte[bufferInfo.size]; @@ -117,7 +127,6 @@ public class AvcEncoder { Log.e(TAG, "Output format changed : " + outBufId); return; } else if (outBufId == MediaCodec.INFO_TRY_AGAIN_LATER) { - Log.e(TAG, "Codec waiting : " + outBufId); return; } else { Log.e(TAG, "Unknown outBufId : " + outBufId); diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java index 12993c7..5401b41 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCamera.java @@ -23,9 +23,7 @@ import static android.hardware.Camera.Parameters.FOCUS_MODE_INFINITY; import static android.hardware.Camera.Parameters.FOCUS_MODE_MACRO; import android.graphics.ImageFormat; -import android.graphics.Rect; import android.graphics.SurfaceTexture; -import android.graphics.YuvImage; import android.hardware.Camera; import android.opengl.GLES11Ext; import android.util.Log; @@ -34,7 +32,6 @@ import android.view.SurfaceHolder; import com.huawei.cloudphone.virtualdevice.common.IVirtualDeviceDataListener; import com.huawei.cloudphone.virtualdevice.common.ParamBundle; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Arrays; import java.util.HashSet; @@ -142,7 +139,7 @@ public class VirtualCamera implements Camera.PreviewCallback { mAvcCodec = new AvcEncoder(mWidth, mHeight, param.getPreviewFrameRate(), mBitrate , new H264DataHandler()); mH264Buffer = new byte[mWidth * mHeight * 3 / 2]; - mAvcCodec.getIFrame(); + mAvcCodec.flush(); } mCamera.startPreview(); return 0; @@ -268,6 +265,12 @@ public class VirtualCamera implements Camera.PreviewCallback { return param; } + public void generateKeyFrame() { + if (mAvcCodec != null) { + mAvcCodec.generateKeyFrame(); + } + } + class H264DataHandler implements AvcEncoderDataHandler { @Override public void handleData(byte[] data, int length) { diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java index 606eae6..6619a3d 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/VirtualCameraManager.java @@ -210,6 +210,11 @@ public class VirtualCameraManager extends VirtualDeviceManager { mVirtualDeviceProtocol.sendMsg(rspHeader, rspBody, CAMERA_DATA); } + public void generateKeyFrame() { + if (mVirtualCamera != null) { + mVirtualCamera.generateKeyFrame(); + } + } class CameraDataListener implements IVirtualDeviceDataListener { @Override diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/common/VirtualDeviceProtocol.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/common/VirtualDeviceProtocol.java index ae769ef..579ff7e 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/common/VirtualDeviceProtocol.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/common/VirtualDeviceProtocol.java @@ -82,6 +82,13 @@ public class VirtualDeviceProtocol { mContext.registerReceiver(mPermissionResultReceiver, intentFilter); } + public VirtualDeviceManager getVirtualDeviceManager(short devType) { + if (virtualDeviceManagers == null) { + return null; + } + return virtualDeviceManagers.get(devType); + } + public void processMsg(MsgHeader header, byte[] body) { VirtualDeviceManager virtualDeviceManager = virtualDeviceManagers.get(header.mDeviceType); if (virtualDeviceManager == null) { -- Gitee From dbc734cae8b134ca378b212ad8c3a263551892c7 Mon Sep 17 00:00:00 2001 From: wangshuo <584363327@qq.com> Date: Fri, 17 Nov 2023 11:39:24 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=91=84=E5=83=8F=E5=A4=B4=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E6=8F=90=E4=BE=9B=E5=BC=BA=E5=88=B6i=E5=B8=A7?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java index b54df3c..5087ba9 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java @@ -73,6 +73,7 @@ public class AvcEncoder { mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat); mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate); + mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL); mediaFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR); try { -- Gitee From 37ee3842c59e909a8f2709e6704d670f36036c63 Mon Sep 17 00:00:00 2001 From: wangshuo <584363327@qq.com> Date: Fri, 17 Nov 2023 17:45:46 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E9=97=B4=E9=9A=94150=E5=B8=A7=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E8=A7=A6=E5=8F=91=E7=94=9F=E6=88=90I=E5=B8=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cloudphone/virtualdevice/camera/AvcEncoder.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java index 5087ba9..f0b3bd9 100644 --- a/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java +++ b/cloudphone/src/main/java/com/huawei/cloudphone/virtualdevice/camera/AvcEncoder.java @@ -36,7 +36,7 @@ public class AvcEncoder { private static final String MIME_TYPE = "video/avc"; private static final int TIMEOUT_USC = 1000; private static final int MICROSECONDS_PER_SECOND = 1000000; - private static final int I_FRAME_INTERVAL = 15; + private static final int I_FRAME_INTERVAL = 150; private static int colorFormat = COLOR_FormatYUV420Planar; private MediaCodec mediaCodec; @@ -45,6 +45,7 @@ public class AvcEncoder { private int mFrameRate; private long genIndex; private byte[] configByte; + private boolean isRequestKeyFrame = false; private AvcEncoderDataHandler dataHandler; public void flush() { @@ -103,6 +104,10 @@ public class AvcEncoder { inputBuffer(frame); int outBufLen = 0; while (true) { + if (genIndex % I_FRAME_INTERVAL == 0 && !isRequestKeyFrame) { + generateKeyFrame(); + isRequestKeyFrame = true; + } MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); int outBufId = mediaCodec.dequeueOutputBuffer(bufferInfo, TIMEOUT_USC); if (outBufId >= 0) { @@ -124,6 +129,7 @@ public class AvcEncoder { dataHandler.handleData(encodeBuff, outBufLen); } mediaCodec.releaseOutputBuffer(outBufId, false); + isRequestKeyFrame = false; } else if (outBufId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { Log.e(TAG, "Output format changed : " + outBufId); return; -- Gitee