diff --git a/impl/kfc/kfc_register_obj.h b/impl/kfc/kfc_register_obj.h index fafc5bfe3a09b0dac9aeffb5cf2e5b93f4bb96b7..aee06e79e21e58d270f28718e7b851321505ef67 100644 --- a/impl/kfc/kfc_register_obj.h +++ b/impl/kfc/kfc_register_obj.h @@ -16,9 +16,43 @@ #define LIB_KFC_REGISTER_OBJ_H #include "kernel_operator.h" -#ifdef ASCENDC_CPU_DEBUG -#if __CCE_AICORE__ == 220 -#ifdef ASCENDC_CUBE_ONLY +namespace AscendC { +constexpr int8_t CUBEOBJ_MIX_MODE = 2; // 0 no, 1 all, 2 mix +template struct CubeObjs {}; + +template __aicore__ inline CubeObjs GetObjType(Args...); // noexception + +template struct GetCubeObjConfig; +template struct GetCubeObjConfig> { + static constexpr int8_t headMixDualMasterValue = GetCubeObjConfig>::enableMixDualMasterValue; + static constexpr int8_t headABShareValue = GetCubeObjConfig>::enableABShareValue; + static constexpr int8_t tailMixDualMasterValue = GetCubeObjConfig>::enableMixDualMasterValue; + static constexpr int8_t tailABShareValue = GetCubeObjConfig>::enableABShareValue; + + static constexpr int8_t enableMixDualMasterValue = (headMixDualMasterValue == -1) ? + tailMixDualMasterValue : + (tailMixDualMasterValue == -1) ? + headMixDualMasterValue : + (headMixDualMasterValue == tailMixDualMasterValue) ? headMixDualMasterValue : CUBEOBJ_MIX_MODE; + static constexpr int8_t enableABShareValue = (headABShareValue == -1) ? + tailABShareValue : + (tailABShareValue == -1) ? headABShareValue : (headABShareValue == tailABShareValue) ? headABShareValue : CUBEOBJ_MIX_MODE; +}; +template struct GetCubeObjConfig> { + static constexpr int8_t enableMixDualMasterValue = T::enableMixDualMaster; + static constexpr int8_t enableABShareValue = T::enableABShare; +}; + +template struct GetCubeObjConfig> { + static constexpr int8_t enableMixDualMasterValue = -1; + static constexpr int8_t enableABShareValue = -1; +}; + +template <> struct GetCubeObjConfig> { + static constexpr int8_t enableMixDualMasterValue = -1; + static constexpr int8_t enableABShareValue = -1; +}; + template __aicore__ static T* GetCurTiling(T* t, Args&&... b) { return t; @@ -46,129 +80,105 @@ __aicore__ inline void InitCurObj(AscendC::TPipe* tpipe, T& a, Args&&... b) } } } +} +#ifdef ASCENDC_CPU_DEBUG +#if __CCE_AICORE__ == 220 +#ifdef ASCENDC_CUBE_ONLY #define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - InitCurObj(tpipe, __VA_ARGS__) + AscendC::InitCurObj(tpipe, __VA_ARGS__) #else -#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - if ASCEND_IS_AIC { \ - AscendC::KfcServer server; \ - server.Init(workspace); \ - server.InitObj(tpipe, __VA_ARGS__); \ - while (server.isRun()) { \ - server.Run(__VA_ARGS__); \ - }; \ - server.Quit(); \ - return; \ - } \ - AscendC::KfcCommClient __kfcClient__(workspace, AscendC::GetSubBlockIdx()); \ - AscendC::g_kfcClient = &__kfcClient__; \ - AscendC::SetMatrixKfc(tpipe, &__kfcClient__, 0, workspace, __VA_ARGS__) +#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ + using ASCubeObjConfig = AscendC::GetCubeObjConfig; \ + static_assert(ASCubeObjConfig::enableABShareValue != AscendC::CUBEOBJ_MIX_MODE, \ + "If both aType ibshare and bType ibshare are set to true, the values must " \ + "be the same for all cube objects."); \ + constexpr int8_t asEnableMixDualMaster = ASCubeObjConfig::enableMixDualMasterValue; \ + static_assert(asEnableMixDualMaster != AscendC::CUBEOBJ_MIX_MODE, \ + "enableMixDualMaster must be consistent for all cube objects."); \ + if ASCEND_IS_AIC { \ + AscendC::KfcServer server; \ + server.Init(workspace); \ + server.InitObj(tpipe, __VA_ARGS__); \ + if constexpr (!asEnableMixDualMaster) { \ + while (server.isRun()) { \ + server.Run(__VA_ARGS__); \ + }; \ + server.Quit(); \ + return; \ + } \ + } \ + AscendC::KfcCommClient __kfcClient__(workspace, AscendC::GetSubBlockIdx()); \ + if ASCEND_IS_AIV { \ + if constexpr (!asEnableMixDualMaster) { \ + AscendC::g_kfcClient = &__kfcClient__; \ + } \ + AscendC::SetMatrixKfc(tpipe, &__kfcClient__, 0, workspace, __VA_ARGS__); \ + } #endif #else -template __aicore__ static T* GetCurTiling(T* t, Args&&... b) -{ - return t; -} - -template -__aicore__ inline void InitCurObjSkip(AscendC::TPipe* tpipe, T* a, - Args&&... b) -{ - InitCurObj(tpipe, b...); -} - -template -__aicore__ inline void InitCurObj(AscendC::TPipe* tpipe, T& a, Args&&... b) -{ - ASSERT(tpipe != nullptr && "tpipe cannot be nullptr"); - if constexpr (sizeof...(b) == 0) { - SetTPipe(a, tpipe); - } else { - auto tiling = GetCurTiling(b...); - a.Init(tiling, tpipe); - if constexpr (sizeof...(b) > 1) { - InitCurObjSkip(tpipe, b...); - } - } -} - #define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - InitCurObj(tpipe, __VA_ARGS__) + AscendC::InitCurObj(tpipe, __VA_ARGS__) #endif #else #ifdef __DAV_C220_CUBE__ #ifdef ASCENDC_CUBE_ONLY -template __aicore__ static T* GetCurTiling(T* t, Args&&... b) -{ - return t; -} - -template -__aicore__ inline void InitCurObjSkip(AscendC::TPipe* tpipe, T* a, - Args&&... b) -{ - InitCurObj(tpipe, b...); -} - -template -__aicore__ inline void InitCurObj(AscendC::TPipe* tpipe, T& a, Args&&... b) -{ - ASSERT(tpipe != nullptr && "tpipe cannot be nullptr"); - if constexpr (sizeof...(b) == 0) { - SetTPipe(a, tpipe); - } else { - auto tiling = GetCurTiling(b...); - a.SetSubBlockIdx(0); - a.Init(tiling, tpipe); - if constexpr (sizeof...(b) > 1) { - InitCurObjSkip(tpipe, b...); - } - } -} - #ifdef ASCENDC_TIME_STAMP_ON #define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - InitCurObj(tpipe, __VA_ARGS__); \ + AscendC::InitCurObj(tpipe, __VA_ARGS__); \ AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_SERVER_OBJ)) #else #define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - InitCurObj(tpipe, __VA_ARGS__) + AscendC::InitCurObj(tpipe, __VA_ARGS__) #endif #define REGIST_CUBE_OBJ_REMOTE(tpipe, workspace, ...) #else #ifdef ASCENDC_TIME_STAMP_ON -#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - if ASCEND_IS_AIC { \ - AscendC::KfcServer server; \ - AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_SERVER)); \ - server.Init(workspace); \ - AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_SERVER_INIT)); \ - server.InitObj(tpipe, __VA_ARGS__); \ - AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_SERVER_OBJ)); \ - while (server.isRun()) { \ - server.Run(__VA_ARGS__); \ - }; \ - server.Quit(); \ - return; \ +#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ + using ASCubeObjConfig = AscendC::GetCubeObjConfig; \ + static_assert(ASCubeObjConfig::enableABShareValue != AscendC::CUBEOBJ_MIX_MODE, \ + "If both aType ibshare and bType ibshare are set to true, the values must " \ + "be the same for all cube objects."); \ + constexpr int8_t asEnableMixDualMaster = ASCubeObjConfig::enableMixDualMasterValue; \ + static_assert(asEnableMixDualMaster != AscendC::CUBEOBJ_MIX_MODE, \ + "enableMixDualMaster must be consistent for all cube objects."); \ + AscendC::KfcServer server; \ + AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_SERVER)); \ + server.Init(workspace); \ + AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_SERVER_INIT)); \ + server.InitObj(tpipe, __VA_ARGS__); \ + AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_SERVER_OBJ)); \ + if constexpr (!asEnableMixDualMaster) { \ + while (server.isRun()) { \ + server.Run(__VA_ARGS__); \ + }; \ + server.Quit(); \ + return; \ } #else -#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - if ASCEND_IS_AIC { \ - AscendC::KfcServer server; \ - server.Init(workspace); \ - server.InitObj(tpipe, __VA_ARGS__); \ - while (server.isRun()) { \ - server.Run(__VA_ARGS__); \ - }; \ - server.Quit(); \ - return; \ +#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ + using ASCubeObjConfig = AscendC::GetCubeObjConfig; \ + static_assert(ASCubeObjConfig::enableABShareValue != AscendC::CUBEOBJ_MIX_MODE, \ + "If both aType ibshare and bType ibshare are set to true, the values must " \ + "be the same for all cube objects."); \ + constexpr int8_t asEnableMixDualMaster = ASCubeObjConfig::enableMixDualMasterValue; \ + static_assert(asEnableMixDualMaster != AscendC::CUBEOBJ_MIX_MODE, \ + "enableMixDualMaster must be consistent for all cube objects."); \ + AscendC::KfcServer server; \ + server.Init(workspace); \ + server.InitObj(tpipe, __VA_ARGS__); \ + if constexpr (!asEnableMixDualMaster) { \ + while (server.isRun()) { \ + server.Run(__VA_ARGS__); \ + }; \ + server.Quit(); \ + return; \ } #endif #endif @@ -179,58 +189,55 @@ __aicore__ inline void InitCurObj(AscendC::TPipe* tpipe, T& a, Args&&... b) return #else #ifdef ASCENDC_TIME_STAMP_ON -#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - AscendC::KfcCommClient __kfcClient__(workspace, GetSubBlockIdx()); \ - g_kfcClient = &__kfcClient__; \ +#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ + using ASCubeObjConfig = AscendC::GetCubeObjConfig; \ + static_assert(ASCubeObjConfig::enableABShareValue != AscendC::CUBEOBJ_MIX_MODE, \ + "If both aType ibshare and bType ibshare are set to true, the values must " \ + "be the same for all cube objects."); \ + constexpr int8_t asEnableMixDualMaster = ASCubeObjConfig::enableMixDualMasterValue; \ + static_assert(asEnableMixDualMaster != AscendC::CUBEOBJ_MIX_MODE, \ + "enableMixDualMaster must be consistent for all cube objects."); \ + AscendC::KfcCommClient __kfcClient__(workspace, AscendC::GetSubBlockIdx()); \ AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_CLIENT_KFC)); \ - AscendC::SetMatrixKfc(tpipe, &__kfcClient__, 0, workspace, __VA_ARGS__); \ + if constexpr (!asEnableMixDualMaster) { \ + AscendC::g_kfcClient = &__kfcClient__; \ + } \ + AscendC::SetMatrixKfc(tpipe, &__kfcClient__, 0, workspace, __VA_ARGS__); \ AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_MATRIX_KFC)); \ - AscendC::WaitEvent(matmul::WORKSPACE_SYNC_ID); \ + if constexpr (!asEnableMixDualMaster) { \ + AscendC::WaitEvent(Gemm::WORKSPACE_SYNC_ID); \ + } \ AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_WAIT_EVE)) + #else -#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - AscendC::KfcCommClient __kfcClient__(workspace, AscendC::GetSubBlockIdx()); \ - AscendC::g_kfcClient = &__kfcClient__; \ - AscendC::SetMatrixKfc(tpipe, &__kfcClient__, 0, workspace, __VA_ARGS__); \ - AscendC::WaitEvent(matmul::WORKSPACE_SYNC_ID) +#define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ + using ASCubeObjConfig = AscendC::GetCubeObjConfig; \ + static_assert(ASCubeObjConfig::enableABShareValue != AscendC::CUBEOBJ_MIX_MODE, \ + "If both aType ibshare and bType ibshare are set to true, the values must " \ + "be the same for all cube objects."); \ + constexpr int8_t asEnableMixDualMaster = ASCubeObjConfig::enableMixDualMasterValue; \ + static_assert(asEnableMixDualMaster != AscendC::CUBEOBJ_MIX_MODE, \ + "enableMixDualMaster must be consistent for all cube objects."); \ + AscendC::KfcCommClient __kfcClient__(workspace, AscendC::GetSubBlockIdx()); \ + if constexpr (!asEnableMixDualMaster) { \ + AscendC::g_kfcClient = &__kfcClient__; \ + } \ + AscendC::SetMatrixKfc(tpipe, &__kfcClient__, 0, workspace, __VA_ARGS__); \ + if constexpr (!asEnableMixDualMaster) { \ + AscendC::WaitEvent(Gemm::WORKSPACE_SYNC_ID); \ + } + #endif #endif #else -template __aicore__ static T* GetCurTiling(T* t, Args&&... b) -{ - return t; -} - -template -__aicore__ inline void InitCurObjSkip(AscendC::TPipe* tpipe, T* a, - Args&&... b) -{ - InitCurObj(tpipe, b...); -} - -template -__aicore__ inline void InitCurObj(AscendC::TPipe* tpipe, T& a, Args&&... b) -{ - ASSERT(tpipe != nullptr && "tpipe cannot be nullptr"); - if constexpr (sizeof...(b) == 0) { - SetTPipe(a, tpipe); - } else { - auto tiling = GetCurTiling(b...); - a.Init(tiling, tpipe); - if constexpr (sizeof...(b) > 1) { - InitCurObjSkip(tpipe, b...); - } - } -} - #ifdef ASCENDC_TIME_STAMP_ON #define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - InitCurObj(tpipe, __VA_ARGS__); \ + AscendC::InitCurObj(tpipe, __VA_ARGS__); \ AscendC::AscendCTimeStamp(static_cast(AscendC::TimeStampId::TIME_STAMP_MATMUL_OBJ)) #else #define REGIST_CUBE_OBJ(tpipe, workspace, ...) \ - InitCurObj(tpipe, __VA_ARGS__) + AscendC::InitCurObj(tpipe, __VA_ARGS__) #endif #endif #endif