diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 27952a401fd08c00d86759e274f43b0b9a56b0b4..0bb800cf8d6c3ec8449975960d1e9aceb00d0e8e 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -42,6 +42,11 @@ load( "if_mkl_ml", "mkl_deps", ) +load( + "//third_party/KDNN:build_defs.bzl", + "if_enable_kdnn", + "kdnn_deps", +) load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") load( "@local_config_rocm//rocm:build_defs.bzl", @@ -4109,10 +4114,8 @@ tf_kernel_library( "//conditions:default": [], }) + mkl_deps() + if_cuda([ "//tensorflow/core/platform/default/build_config:cublas_plugin", - ]) + select({ - "@org_tensorflow//tensorflow:linux_aarch64": ["//third_party/KDNN:kdnn_adapter"], - "//conditions:default": [], - }), + ]) + kdnn_deps(), + copts = if_enable_kdnn(["-DENABLE_KDNN=1"]), ) tf_mkl_kernel_library( diff --git a/tensorflow/core/kernels/matmul_op.cc b/tensorflow/core/kernels/matmul_op.cc index 511b791dae087d747df01dd0916eab0264c8d85f..90e5aaa3166a42b28bfcdf7d6165cb289ea4773a 100644 --- a/tensorflow/core/kernels/matmul_op.cc +++ b/tensorflow/core/kernels/matmul_op.cc @@ -32,7 +32,7 @@ limitations under the License. #include "tensorflow/core/platform/stream_executor.h" #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#if defined(__aarch64__) +#if defined(ENABLE_KDNN) #include "kdnn_adapter.h" #endif @@ -446,13 +446,17 @@ template class MatMulOp : public OpKernel { public: explicit MatMulOp(OpKernelConstruction* ctx) - : OpKernel(ctx), algorithms_set_already_(false), kp_enable(std::getenv("KP_ENABLE")) { + : OpKernel(ctx), algorithms_set_already_(false), kdnn_enable(true) { OP_REQUIRES_OK(ctx, ctx->GetAttr("transpose_a", &transpose_a_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("transpose_b", &transpose_b_)); LaunchMatMul::GetBlasGemmAlgorithm( ctx, &algorithms_, &algorithms_set_already_); use_autotune_ = MatmulAutotuneEnable(); + char* kdnn_env = std::getenv("KDNN_ENABLE"); + if (kdnn_env && std::string(kdnn_env) == "off") { + kdnn_enable = false; + } } void Compute(OpKernelContext* ctx) override { @@ -521,8 +525,9 @@ class MatMulOp : public OpKernel { FloatToBFloat16(out_float.flat().data(), out->flat().data(), out->NumElements()); } -#if defined(__aarch64__) - else if (kp_enable && std::is_same::value && !transpose_a_ && !transpose_b_) { +#if defined(ENABLE_KDNN) + else if (kdnn_enable && std::is_same::value && + !transpose_a_ && !transpose_b_) { kdnnGemm(ctx, a, b, out, transpose_a_, transpose_b_); } #endif @@ -538,7 +543,7 @@ class MatMulOp : public OpKernel { bool use_autotune_; bool transpose_a_; bool transpose_b_; - char *kp_enable; + bool kdnn_enable; }; namespace functor { diff --git a/third_party/KDNN/BUILD b/third_party/KDNN/BUILD index a4370540dfdc3c03f208a8feab40adc3144704ce..bcf86c35927d0678badb8acb3f44eb3d57dd1815 100644 --- a/third_party/KDNN/BUILD +++ b/third_party/KDNN/BUILD @@ -1,13 +1,20 @@ licenses(["notice"]) +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +config_setting( + name = "enable_kdnn", + define_values = { + "enable_kdnn": "true", + }, + visibility = ["//visibility:public"], +) + cc_library( name = "kdnn", - hdrs = glob(["kdnn_include/**/*.hpp"]), - includes = ["kdnn_include"], - srcs = ["kdnn_lib/libkdnn.so"], - copts = ["-fopenmp"], - linkopts = ["-fopenmp"], - alwayslink = 1, + hdrs = glob(["src/dnn/include/**/*.hpp"]), + includes = ["src/dnn/include"], + srcs = glob(["src/dnn/src/*.cpp", "src/dnn/src/*.hpp"]), ) cc_library( @@ -17,3 +24,9 @@ cc_library( visibility = ["//visibility:public"], deps = [":kdnn"], ) + +bzl_library( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/third_party/KDNN/build_defs.bzl b/third_party/KDNN/build_defs.bzl new file mode 100644 index 0000000000000000000000000000000000000000..da1748fc95a8780d8ec1584ef40799d0f6dc8346 --- /dev/null +++ b/third_party/KDNN/build_defs.bzl @@ -0,0 +1,24 @@ +def if_enable_kdnn(if_true, if_false = []): + """Shorthand to select() if we are building with KDNN and KDNN is enabled. + + This is only effective when built with KDNN. + + Args: + if_true: expression to evaluate if building with KDNN and KDNN is enabled + if_false: expression to evaluate if building without KDNN or KDNN is not enabled. + + Returns: + A select evaluating to either if_true or if_false as appropriate. + """ + return select({ + "@org_tensorflow//third_party/KDNN:enable_kdnn": if_true, + "//conditions:default": if_false, + }) + +def kdnn_deps(): + """Shorthand for select() to pull in the correct set of KDNN library deps. + """ + return select({ + "@org_tensorflow//third_party/KDNN:enable_kdnn": ["//third_party/KDNN:kdnn_adapter"], + "//conditions:default": [], + }) \ No newline at end of file diff --git a/third_party/KDNN/kdnn_adapter.h b/third_party/KDNN/kdnn_adapter.h index 97447f2ee4ddee78a4c5a8cf32b1745a9f3fae54..6a5adad9e3aef86ff581727b1732f533bebb0103 100644 --- a/third_party/KDNN/kdnn_adapter.h +++ b/third_party/KDNN/kdnn_adapter.h @@ -11,8 +11,6 @@ inline void kdnnGemm(OpKernelContext* ctx, const Tensor& a, const Tensor& b, Ten const float *A = a.flat().data(); const float *B = b.flat().data(); float *C = out->flat().data(); - float alpha = 1.0; - float beta = 0.0; // intra_op thread_pool thread::ThreadPool* thread_pool = ctx->device() @@ -23,7 +21,7 @@ inline void kdnnGemm(OpKernelContext* ctx, const Tensor& a, const Tensor& b, Ten const KDNN::TensorInfo weightsInfo = {{k, n}, KDNN::Element::TypeT::F32, KDNN::Layout::AB}; const KDNN::TensorInfo dstInfo = {{m, n}, KDNN::Element::TypeT::F32, KDNN::Layout::AB}; KDNN::Gemm gemm(srcInfo, weightsInfo, dstInfo, &eigen_tp); - gemm.Run(A, B, C, alpha, beta); + gemm.Run(A, B, C); } }// namespace tensorflow \ No newline at end of file diff --git a/third_party/KDNN/kdnn_include/kdnn.hpp b/third_party/KDNN/kdnn_include/kdnn.hpp deleted file mode 100644 index b66901375b7912da52730bf8edbf82af096d4dcd..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/kdnn.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Used header files - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_HPP -#define KDNN_HPP - -#include "types/kdnn_types.hpp" -#include "types/kdnn_data_type.hpp" -#include "types/kdnn_layout.hpp" -#include "types/kdnn_propagation_type.hpp" -#include "types/kdnn_shape.hpp" -#include "types/kdnn_types.hpp" - -#include "service/kdnn_err_codes.hpp" -#include "service/kdnn_exception.hpp" -#include "service/kdnn_service.hpp" -#include "service/kdnn_threading.hpp" - -#include "operations/kdnn_batch_normalization.hpp" -#include "operations/kdnn_binary.hpp" -#include "operations/kdnn_convolution.hpp" -#include "operations/kdnn_deconvolution.hpp" -#include "operations/kdnn_eltwise.hpp" -#include "operations/kdnn_gemm.hpp" -#include "operations/kdnn_group_normalization.hpp" -#include "operations/kdnn_inner_product.hpp" -#include "operations/kdnn_layer_normalization.hpp" -#include "operations/kdnn_lrn.hpp" -#include "operations/kdnn_pooling.hpp" -#include "operations/kdnn_prelu.hpp" -#include "operations/kdnn_reduction.hpp" -#include "operations/kdnn_reorder.hpp" -#include "operations/kdnn_shuffle.hpp" -#include "operations/kdnn_softmax.hpp" -#include "operations/kdnn_sum.hpp" -#include "operations/kdnn_resampling.hpp" -#include "operations/kdnn_concat.hpp" -#include "operations/kdnn_rnn.hpp" - -#endif // KDNN_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_batch_normalization.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_batch_normalization.hpp deleted file mode 100644 index 3846eada2001108075324dc96497197177618a37..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_batch_normalization.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Batch Normalization - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_BATCH_NORMALIZATION_HPP -#define KDNN_BATCH_NORMALIZATION_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "types/kdnn_normalization.hpp" -#include "service/kdnn_err_codes.hpp" -#include "types/kdnn_propagation_type.hpp" - -namespace KDNN { - -namespace Detail { - -class BatchNormalizationLayerFWDImpl; -class BatchNormalizationLayerBWDImpl; - -} // Detail - -class KDNN_API_PUBLIC BatchNormalizationLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const Propagation &propKind, const TensorInfo &srcInfo, const TensorInfo &statsInfo, - const TensorInfo &scaleShiftInfo, const TensorInfo &dstInfo, - NormalizationFlags flags) noexcept; - BatchNormalizationLayerFWD(const Propagation &propKind, const TensorInfo &srcInfo, const TensorInfo &statsInfo, - const TensorInfo &scaleShiftInfo, const TensorInfo &dstInfo, NormalizationFlags flags) noexcept(false); - BatchNormalizationLayerFWD(const BatchNormalizationLayerFWD &other) noexcept(false); - BatchNormalizationLayerFWD(BatchNormalizationLayerFWD &&other) noexcept; - BatchNormalizationLayerFWD& operator=(const BatchNormalizationLayerFWD &other) noexcept(false); - BatchNormalizationLayerFWD& operator=(BatchNormalizationLayerFWD &&other) noexcept; - void Run(const void *src, void *dst, const float *scale, const float *shift, float *mean, float *variance, - bool saveStats, const float eps, void *ws = nullptr) const noexcept(false); - ~BatchNormalizationLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC BatchNormalizationLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const Propagation &propKind, const TensorInfo &srcInfo, const TensorInfo &diffDstInfo, - const TensorInfo &statsInfo, const TensorInfo &scaleShiftInfo, const TensorInfo &diffSrcInfo, - const TensorInfo &diffScaleShiftInfo, NormalizationFlags flags) noexcept; - BatchNormalizationLayerBWD(const Propagation &propKind, const TensorInfo &srcInfo, const TensorInfo &diffDstInfo, - const TensorInfo &statsInfo, const TensorInfo &scaleShiftInfo, const TensorInfo &diffSrcInfo, - const TensorInfo &diffScaleShiftInfo, NormalizationFlags flags) noexcept(false); - BatchNormalizationLayerBWD(const BatchNormalizationLayerBWD &other) noexcept(false); - BatchNormalizationLayerBWD(BatchNormalizationLayerBWD &&other) noexcept; - BatchNormalizationLayerBWD& operator=(const BatchNormalizationLayerBWD &other) noexcept(false); - BatchNormalizationLayerBWD& operator=(BatchNormalizationLayerBWD &&other) noexcept; - void Run(const void *src, const void *diffDst, const float *mean, const float *variance, const float *scale, - void *diffSrc, float *diffScale, float *diffShift, const float eps, - const void *ws = nullptr) const noexcept(false); - ~BatchNormalizationLayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_BATCH_NORMALIZATION_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_binary.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_binary.hpp deleted file mode 100644 index 64ce7eb656f1134bed83f97de573c1221f7013bf..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_binary.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: binary correlation function - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_BINARY_HPP -#define KDNN_BINARY_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -enum class BinaryFunction { - UNIMPLEMENTED, - ADD, - DIV, - MAX, - MIN, - MUL, - SUB, - GE, - GT, - LE, - LT, - EQ, - NE -}; - -namespace Detail { - -class BinaryLayerImpl; - -} // Detail - -class KDNN_API_PUBLIC BinaryLayer final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &src0Info, const TensorInfo &src1Info, - const TensorInfo &dstInfo, BinaryFunction op) noexcept; - BinaryLayer(const TensorInfo &src0Info, const TensorInfo &src1Info, - const TensorInfo &dstInfo, BinaryFunction op) noexcept(false); - BinaryLayer(const BinaryLayer &other) noexcept(false); - BinaryLayer(BinaryLayer &&other) noexcept; - BinaryLayer& operator=(const BinaryLayer &other) noexcept(false); - BinaryLayer& operator=(BinaryLayer &&other) noexcept; - void Run(const void *src0, const void *src1, void *dst, - const float scale0 = 1.0f, const float scale1 = 1.0f) const noexcept(false); - ~BinaryLayer() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_BINARY_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_concat.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_concat.hpp deleted file mode 100644 index addf854c77f91091658f5a4f4c182150e8111e44..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_concat.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Concat - * Author: KPL - * Create: 2024-07-01 - * Notes: NA - */ - -#ifndef KDNN_CONCAT_HPP -#define KDNN_CONCAT_HPP - -#include -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class ConcatLayerImpl; - -} // Detail - -class KDNN_API_PUBLIC ConcatLayer final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const std::vector &srcInfo, - const int concatDim, const TensorInfo &dstInfo) noexcept; - ConcatLayer(const std::vector &srcInfo, const int concatDim, - const TensorInfo &dstInfo) noexcept(false); - ConcatLayer(const ConcatLayer &other) noexcept(false); - ConcatLayer(ConcatLayer &&other) noexcept; - ConcatLayer& operator=(const ConcatLayer &other) noexcept(false); - ConcatLayer& operator=(ConcatLayer &&other) noexcept; - void Run(const void **src, void *dst) const noexcept(false); - ~ConcatLayer() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_CONCAT_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_convolution.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_convolution.hpp deleted file mode 100644 index 0c9cb49fc16d1d15539eafd8e927d1924f5b704f..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_convolution.hpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: convolution impl - * Author: KPL - * Create: 2024-04-27 - * Notes: NA - */ - -#ifndef KDNN_CONVOLUTION_HPP -#define KDNN_CONVOLUTION_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -enum class ConvolutionAlgorithm { - UNIMPLEMENTED, - AUTO, - DIRECT, - WINOGRAD -}; - -namespace Detail { - -class ConvolutionLayerFWDImpl; -class ConvolutionLayerBWDDataImpl; -class ConvolutionLayerBWDWeightsImpl; -class ConvolutionJITFWDImpl; - -} // Detail - -class KDNN_API_PUBLIC ConvolutionLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept; - ConvolutionLayerFWD(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, - const Shape &strides, const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept(false); - ConvolutionLayerFWD(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept(false); - ConvolutionLayerFWD(const ConvolutionLayerFWD &other) noexcept(false); - ConvolutionLayerFWD(ConvolutionLayerFWD &&other) noexcept; - ConvolutionLayerFWD& operator=(const ConvolutionLayerFWD &other) noexcept(false); - ConvolutionLayerFWD& operator=(ConvolutionLayerFWD &&other) noexcept; - void Run(const void *src, const void *wei, void *dst, const void *bia) const noexcept(false); - ~ConvolutionLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC ConvolutionLayerBWDData final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept; - ConvolutionLayerBWDData(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc, - const Shape &strides, const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept(false); - ConvolutionLayerBWDData(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept(false); - ConvolutionLayerBWDData(const ConvolutionLayerBWDData &other) noexcept(false); - ConvolutionLayerBWDData(ConvolutionLayerBWDData &&other) noexcept; - ConvolutionLayerBWDData& operator=(const ConvolutionLayerBWDData &other) noexcept(false); - ConvolutionLayerBWDData& operator=(ConvolutionLayerBWDData &&other) noexcept; - void Run(const void *diffDst, const void *wei, void *diffSrc) const noexcept(false); - ~ConvolutionLayerBWDData() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC ConvolutionLayerBWDWeights final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept; - ConvolutionLayerBWDWeights(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias, - const Shape &strides, const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept(false); - ConvolutionLayerBWDWeights(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - ConvolutionAlgorithm alg) noexcept(false); - ConvolutionLayerBWDWeights(const ConvolutionLayerBWDWeights &other) noexcept(false); - ConvolutionLayerBWDWeights(ConvolutionLayerBWDWeights &&other) noexcept; - ConvolutionLayerBWDWeights& operator=(const ConvolutionLayerBWDWeights &other) noexcept(false); - ConvolutionLayerBWDWeights& operator=(ConvolutionLayerBWDWeights &&other) noexcept; - void Run(const void *diffDst, const void *src, void *diffWei, void *diffBias) const noexcept(false); - ~ConvolutionLayerBWDWeights() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC ConvolutionJITFWD final { -public: - static Status ValidateInput(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, - const Shape &strides, const Shape &dilates, const Shape &paddingL, int kCpuIsa) noexcept; - explicit ConvolutionJITFWD(int isa, const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, KDNN::Shape strides, KDNN::Shape paddingL, - KDNN::Shape dilates, KDNN::Propagation prop, int nthreads, bool isDataLayoutNxc) noexcept(false); - void ExecuteForwardDW(const float *src, const float *wei, - float *dst, const float *bias) noexcept; - void ExecuteForward1D(const float *src, const float *wei, - float *dst, const float *bias) noexcept; - void ExecuteForward2D(const float *src, const float *wei, - float *dst, const float *bias) noexcept; - void ExecuteForward3D(const float *src, const float *wei, - float *dst, const float *bias) noexcept; - ConvolutionJITFWD(const ConvolutionJITFWD &other) noexcept(false); - ConvolutionJITFWD(ConvolutionJITFWD &&other) noexcept; - ConvolutionJITFWD& operator=(const ConvolutionJITFWD &other) noexcept(false); - ConvolutionJITFWD& operator=(ConvolutionJITFWD &&other) noexcept; - bool CreateKernel(); - ~ConvolutionJITFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_CONVOLUTION_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_deconvolution.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_deconvolution.hpp deleted file mode 100644 index 4a4a741bec4dc5de8040a489001ac4395d4416a8..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_deconvolution.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: deconvolution impl - * Author: KPL - * Create: 2024-05-24 - * Notes: NA - */ - -#ifndef KDNN_DECONVOLUTION_HPP -#define KDNN_DECONVOLUTION_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -enum class DeconvolutionAlgorithm { - UNIMPLEMENTED, - DIRECT, - WINOGRAD -}; - -namespace Detail { - -class DeconvolutionLayerFWDImpl; -class DeconvolutionLayerBWDDataImpl; -class DeconvolutionLayerBWDWeightsImpl; - -} // Detail - -class KDNN_API_PUBLIC DeconvolutionLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - DeconvolutionAlgorithm alg) noexcept; - DeconvolutionLayerFWD(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, - const Shape &strides, const Shape &paddingL, const Shape &paddingR, - DeconvolutionAlgorithm alg) noexcept(false); - DeconvolutionLayerFWD(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - DeconvolutionAlgorithm alg) noexcept(false); - DeconvolutionLayerFWD(const DeconvolutionLayerFWD &other) noexcept(false); - DeconvolutionLayerFWD(DeconvolutionLayerFWD &&other) noexcept; - DeconvolutionLayerFWD& operator=(const DeconvolutionLayerFWD &other) noexcept(false); - DeconvolutionLayerFWD& operator=(DeconvolutionLayerFWD &&other) noexcept; - void Run(const void *src, const void *wei, void *dst, const void *bia) const noexcept(false); - ~DeconvolutionLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC DeconvolutionLayerBWDData final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc, const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, DeconvolutionAlgorithm alg) noexcept; - DeconvolutionLayerBWDData(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc, const Shape &strides, const Shape &paddingL, - const Shape &paddingR, DeconvolutionAlgorithm alg) noexcept(false); - DeconvolutionLayerBWDData(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc, const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, DeconvolutionAlgorithm alg) noexcept(false); - DeconvolutionLayerBWDData(const DeconvolutionLayerBWDData &other) noexcept(false); - DeconvolutionLayerBWDData(DeconvolutionLayerBWDData &&other) noexcept; - DeconvolutionLayerBWDData& operator=(const DeconvolutionLayerBWDData &other) noexcept(false); - DeconvolutionLayerBWDData& operator=(DeconvolutionLayerBWDData &&other) noexcept; - void Run(const void *diffDst, const void *wei, void *diffSrc) const noexcept(false); - ~DeconvolutionLayerBWDData() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC DeconvolutionLayerBWDWeights final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - DeconvolutionAlgorithm alg) noexcept; - DeconvolutionLayerBWDWeights(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias, - const Shape &strides, const Shape &paddingL, const Shape &paddingR, - DeconvolutionAlgorithm alg) noexcept(false); - DeconvolutionLayerBWDWeights(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias, - const Shape &strides, const Shape &dilates, - const Shape &paddingL, const Shape &paddingR, - DeconvolutionAlgorithm alg) noexcept(false); - DeconvolutionLayerBWDWeights(const DeconvolutionLayerBWDWeights &other) noexcept(false); - DeconvolutionLayerBWDWeights(DeconvolutionLayerBWDWeights &&other) noexcept; - DeconvolutionLayerBWDWeights& operator=(const DeconvolutionLayerBWDWeights &other) noexcept(false); - DeconvolutionLayerBWDWeights& operator=(DeconvolutionLayerBWDWeights &&other) noexcept; - void Run(const void *diffDst, const void *src, void *diffWei, void *diffBias) const noexcept(false); - ~DeconvolutionLayerBWDWeights() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_DECONVOLUTION_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_eltwise.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_eltwise.hpp deleted file mode 100644 index 1ff9ccfdcf28f62305ef37daa8f2907a319a5fb3..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_eltwise.hpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: mathematical correlation function - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_ELTWISE_HPP -#define KDNN_ELTWISE_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { -#ifndef INFINITY -#define INFINITY (__builtin_inff()) -#endif - -// supported functions -enum class ActivationFunction { - UNIMPLEMENTED, - ABS, - EXP, - LINEAR, - LOG, - RELU, - ROUND, - SQRT, - SQUARE, - TANH, - SIGMOID, - POW, - CLIP, - CLIP_V2, - HARDSIGMOID, - HARDSWISH, - EXP_DST, - LOGISTIC_DST, - TANH_DST, - ELU, - GELU_TANH, - CLIP_V2_DST, - ELU_DST, - RELU_DST, - SQRT_DST, - SWISH, - SRELU, - GELU_ERF, - MISH -}; - -namespace Detail { - -class ActivationLayerImplFWD; -class ActivationLayerImplBWD; - -} // Detail - -class KDNN_API_PUBLIC ActivationLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &dstInfo, - ActivationFunction kind, float alpha = 0.0f, float beta = 0.0f) noexcept; - ActivationLayerFWD(const TensorInfo &srcInfo, const TensorInfo &dstInfo, - ActivationFunction kind, float alpha = 0.0f, float beta = 0.0f) noexcept(false); - ActivationLayerFWD(const ActivationLayerFWD &other) noexcept(false); - ActivationLayerFWD(ActivationLayerFWD &&other) noexcept; - ActivationLayerFWD& operator=(const ActivationLayerFWD &other) noexcept(false); - ActivationLayerFWD& operator=(ActivationLayerFWD &&other) noexcept; - void Run(const void *src, void *dst) const noexcept(false); - ~ActivationLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC ActivationLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &dsInfo, const TensorInfo &ddInfo, - const TensorInfo &srcInfo, ActivationFunction kind, float alpha = 0.0f, - float beta = 0.0f) noexcept; - ActivationLayerBWD(const TensorInfo &dsInfo, const TensorInfo &ddInfo, - const TensorInfo &srcInfo, ActivationFunction kind, float alpha = 0.0f, - float beta = 0.0f) noexcept(false); - ActivationLayerBWD(const ActivationLayerBWD &other) noexcept(false); - ActivationLayerBWD(ActivationLayerBWD &&other) noexcept; - ActivationLayerBWD& operator=(const ActivationLayerBWD &other) noexcept(false); - ActivationLayerBWD& operator=(ActivationLayerBWD &&other) noexcept; - void Run(void *ds, const void *dd, const void *src) const noexcept(false); - ~ActivationLayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_ELTWISE_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_gemm.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_gemm.hpp deleted file mode 100644 index ea4a01a656d0e73aeb3efceb9be48c365e94d6b8..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_gemm.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Gemm Operation - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_GEMM_HPP -#define KDNN_GEMM_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class GemmImpl; - -} // Detail - -class KDNN_API_PUBLIC Gemm final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &aInfo, const TensorInfo &bInfo, - const TensorInfo &cInfo, const TensorInfo &biasInfo) noexcept; - static Status ValidateInput(const TensorInfo &aInfo, const TensorInfo &bInfo, - const TensorInfo &cInfo) noexcept; - Gemm(const TensorInfo &aInfo, const TensorInfo &bInfo, const TensorInfo &cInfo, - const TensorInfo &biasInfo) noexcept(false); - Gemm(const TensorInfo &aInfo, const TensorInfo &bInfo, const TensorInfo &cInfo, - const TensorInfo &biasInfo, void *thread_pool) noexcept(false); - Gemm(const TensorInfo &aInfo, const TensorInfo &bInfo, const TensorInfo &cInfo) noexcept(false); - Gemm(const TensorInfo &aInfo, const TensorInfo &bInfo, - const TensorInfo &cInfo, void *thread_pool) noexcept(false); - Gemm(const Gemm &other) noexcept(false); - Gemm(Gemm &&other) noexcept; - Gemm& operator=(const Gemm &other) noexcept(false); - Gemm& operator=(Gemm &&other) noexcept; - void Run(const void *a, const void *b, void *c, const void *bias, - float alpha = 1.0f, float beta = 0.0f) const noexcept(false); - void Run(const void *a, const void *b, void *c, - float alpha = 1.0f, float beta = 0.0f) const noexcept(false); - ~Gemm() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_GEMM_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_group_normalization.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_group_normalization.hpp deleted file mode 100644 index 1d979c2c1cc9cc5a166a35fc543cd27f6494d82f..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_group_normalization.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Group Normalization - * Author: KPL - * Create: 2024-10-10 - * Notes: NA - */ - -#ifndef KDNN_GROUP_NORMALIZATION_HPP -#define KDNN_GROUP_NORMALIZATION_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class GroupNormalizationLayerFWDImpl; -class GroupNormalizationLayerBWDImpl; - -} // Detail - -class KDNN_API_PUBLIC GroupNormalizationLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &scaleshiftInfo, SizeType groupInfo, - const TensorInfo &dstInfo, NormalizationFlags flags) noexcept; - GroupNormalizationLayerFWD(const TensorInfo &srcInfo, const TensorInfo &scaleshiftInfo, SizeType groupSize, - const TensorInfo &dstInfo, NormalizationFlags flags) noexcept(false); - GroupNormalizationLayerFWD(const GroupNormalizationLayerFWD &other) noexcept(false); - GroupNormalizationLayerFWD(GroupNormalizationLayerFWD &&other) noexcept; - GroupNormalizationLayerFWD& operator=(const GroupNormalizationLayerFWD &other) noexcept(false); - GroupNormalizationLayerFWD& operator=(GroupNormalizationLayerFWD &&other) noexcept; - void Run(const void *src, void *dst, const void *scale, const void *shift, float *mean, float *variance, - bool saveStats, const float eps) const noexcept(false); - ~GroupNormalizationLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC GroupNormalizationLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &statInfo, - const TensorInfo &diffSrcInfo, const TensorInfo &diffDstInfo, - const TensorInfo &scaleShiftInfo, const TensorInfo &diffscaleShiftInfo, - NormalizationFlags flags) noexcept; - GroupNormalizationLayerBWD(const TensorInfo &srcInfo, const TensorInfo &statInfo, - const TensorInfo &diffSrcInfo, const TensorInfo &diffDstInfo, - const TensorInfo &scaleShiftInfo, const TensorInfo &diffscaleShiftInfo, - NormalizationFlags flags) noexcept(false); - GroupNormalizationLayerBWD(const GroupNormalizationLayerBWD &other) noexcept(false); - GroupNormalizationLayerBWD(GroupNormalizationLayerBWD &&other) noexcept; - GroupNormalizationLayerBWD& operator=(const GroupNormalizationLayerBWD &other) noexcept(false); - GroupNormalizationLayerBWD& operator=(GroupNormalizationLayerBWD &&other) noexcept; - void Run(const void *src, const float *mean, const float *variance, const void *diffDst, - const void *scale, void *diffSrc, void *diffScale, - void *diffShift, float eps) const noexcept(false); - ~GroupNormalizationLayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_GROUP_NORMALIZATION_HPP \ No newline at end of file diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_inner_product.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_inner_product.hpp deleted file mode 100644 index e950b77dda160657348059427f58bc92d6e727dd..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_inner_product.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Inner product - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_INNER_PRODUCT_HPP -#define KDNN_INNER_PRODUCT_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class InnerProductLayerFWDImpl; -class InnerProductLayerBWDDataImpl; -class InnerProductLayerBWDWeightsImpl; - -} // Detail - -class KDNN_API_PUBLIC InnerProductLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias) noexcept; - InnerProductLayerFWD(const TensorInfo &src, const TensorInfo &weights, - const TensorInfo &dst, const TensorInfo &bias) noexcept(false); - InnerProductLayerFWD(const InnerProductLayerFWD &other) noexcept(false); - InnerProductLayerFWD(InnerProductLayerFWD &&other) noexcept; - InnerProductLayerFWD& operator=(const InnerProductLayerFWD &other) noexcept(false); - InnerProductLayerFWD& operator=(InnerProductLayerFWD &&other) noexcept; - void Run(const void *src, const void *wei, void *dst, const void *bia = nullptr) const noexcept(false); - ~InnerProductLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC InnerProductLayerBWDData final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc) noexcept; - InnerProductLayerBWDData(const TensorInfo &diffDst, const TensorInfo &weights, - const TensorInfo &diffSrc) noexcept(false); - InnerProductLayerBWDData(const InnerProductLayerBWDData &other) noexcept(false); - InnerProductLayerBWDData(InnerProductLayerBWDData &&other) noexcept; - InnerProductLayerBWDData& operator=(const InnerProductLayerBWDData &other) noexcept(false); - InnerProductLayerBWDData& operator=(InnerProductLayerBWDData &&other) noexcept; - void Run(const void *diffDst, const void *wei, void *diffSrc) const noexcept(false); - ~InnerProductLayerBWDData() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC InnerProductLayerBWDWeights final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias) noexcept; - InnerProductLayerBWDWeights(const TensorInfo &diffDst, const TensorInfo &src, - const TensorInfo &diffWeights, const TensorInfo &diffBias) noexcept(false); - InnerProductLayerBWDWeights(const InnerProductLayerBWDWeights &other) noexcept(false); - InnerProductLayerBWDWeights(InnerProductLayerBWDWeights &&other) noexcept; - InnerProductLayerBWDWeights& operator=(const InnerProductLayerBWDWeights &other) noexcept(false); - InnerProductLayerBWDWeights& operator=(InnerProductLayerBWDWeights &&other) noexcept; - void Run(const void *diffDst, const void *src, void *diffWeights, void *diffBias = nullptr) const noexcept(false); - ~InnerProductLayerBWDWeights() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_INNER_PRODUCT_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_layer_normalization.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_layer_normalization.hpp deleted file mode 100644 index c89536c54b23797e3d0841b2d2e2de8b73c6d645..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_layer_normalization.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Normalization - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_LAYER_NORMALIZATION_HPP -#define KDNN_LAYER_NORMALIZATION_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class NormalizationLayerFWDImpl; -class NormalizationLayerBWDImpl; - -} // Detail - -class KDNN_API_PUBLIC NormalizationLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &statsInfo, - const TensorInfo &scaleshiftInfo, const TensorInfo &dstInfo, - NormalizationFlags flags) noexcept; - NormalizationLayerFWD(const TensorInfo &srcInfo, const TensorInfo &statsInfo, const TensorInfo &scaleshiftInfo, - const TensorInfo &dstInfo, NormalizationFlags flags) noexcept(false); - NormalizationLayerFWD(const NormalizationLayerFWD &other) noexcept(false); - NormalizationLayerFWD(NormalizationLayerFWD &&other) noexcept; - NormalizationLayerFWD& operator=(const NormalizationLayerFWD &other) noexcept(false); - NormalizationLayerFWD& operator=(NormalizationLayerFWD &&other) noexcept; - void Run(const void *src, void *dst, const void *scale, const void *shift, float *mean, float *variance, - bool saveStats, const float eps) const noexcept(false); - ~NormalizationLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC NormalizationLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &statInfo, - const TensorInfo &diffSrcInfo, const TensorInfo &diffDstInfo, - const TensorInfo &scaleShiftInfo, const TensorInfo &diffscaleShiftInfo, - NormalizationFlags flags) noexcept; - NormalizationLayerBWD(const TensorInfo &srcInfo, const TensorInfo &statInfo, - const TensorInfo &diffSrcInfo, const TensorInfo &diffDstInfo, - const TensorInfo &scaleShiftInfo, const TensorInfo &diffscaleShiftInfo, - NormalizationFlags flags) noexcept(false); - NormalizationLayerBWD(const NormalizationLayerBWD &other) noexcept(false); - NormalizationLayerBWD(NormalizationLayerBWD &&other) noexcept; - NormalizationLayerBWD& operator=(const NormalizationLayerBWD &other) noexcept(false); - NormalizationLayerBWD& operator=(NormalizationLayerBWD &&other) noexcept; - void Run(const void *src, const float *mean, const float *variance, const void *diffDst, - const void *scale, void *diffSrc, void *diffScale, - void *diffShift, float eps) const noexcept(false); - ~NormalizationLayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_LAYER_NORMALIZATION_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_lrn.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_lrn.hpp deleted file mode 100644 index 51292d3d8b166847983ec197a099aa013154750f..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_lrn.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Local Response Normalization - * Author: KPL - * Create: 2024-07-29 - * Notes: NA - */ - -#ifndef KDNN_LRN_HPP -#define KDNN_LRN_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -enum class LRNormAlgorithmKind : std::uint32_t { - WITHIN = 0x0U, - ACROSS = 0x1U, -}; - -namespace Detail { - -class LocalResponseNormalizationLayerFWDImpl; -class LocalResponseNormalizationLayerBWDImpl; - -} // Detail - -class KDNN_API_PUBLIC LocalResponseNormalizationLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &dstInfo, float alpha, float beta, float k, - SizeType localSize, LRNormAlgorithmKind algorithm) noexcept; - LocalResponseNormalizationLayerFWD(const TensorInfo &srcInfo, const TensorInfo &dstInfo, float alpha, float beta, - float k, SizeType localSize, LRNormAlgorithmKind algorithm) noexcept(false); - LocalResponseNormalizationLayerFWD(const LocalResponseNormalizationLayerFWD &other) noexcept(false); - LocalResponseNormalizationLayerFWD(LocalResponseNormalizationLayerFWD &&other) noexcept; - LocalResponseNormalizationLayerFWD& operator=(const LocalResponseNormalizationLayerFWD &other) noexcept(false); - LocalResponseNormalizationLayerFWD& operator=(LocalResponseNormalizationLayerFWD &&other) noexcept; - void Run(const void *src, void *dst) const noexcept(false); - ~LocalResponseNormalizationLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC LocalResponseNormalizationLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, - const TensorInfo &diffDstInfo, const TensorInfo &diffSrcInfo, - float alpha, float beta, float k, - SizeType localSize, LRNormAlgorithmKind algorithm) noexcept; - LocalResponseNormalizationLayerBWD(const TensorInfo &srcInfo, - const TensorInfo &diffDstInfo, const TensorInfo &diffSrcInfo, - float alpha, float beta, float k, - SizeType localSize, LRNormAlgorithmKind algorithm) noexcept(false); - LocalResponseNormalizationLayerBWD(const LocalResponseNormalizationLayerBWD &other) noexcept(false); - LocalResponseNormalizationLayerBWD(LocalResponseNormalizationLayerBWD &&other) noexcept; - LocalResponseNormalizationLayerBWD& operator=(const LocalResponseNormalizationLayerBWD &other) noexcept(false); - LocalResponseNormalizationLayerBWD& operator=(LocalResponseNormalizationLayerBWD &&other) noexcept; - void Run(const void *src, const void *diffDst, - void *diffSrc) const noexcept(false); - ~LocalResponseNormalizationLayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_LRN_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_pooling.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_pooling.hpp deleted file mode 100644 index d5e1cb3d5ab3bd70a415407120633613ffbeda11..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_pooling.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Pooling op - * Author: KPL - * Create: 2024-07-19 - * Notes: NA - */ - -#ifndef KDNN_POOLING_HPP -#define KDNN_POOLING_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" -#include "types/kdnn_propagation_type.hpp" - -namespace KDNN { - -enum class PoolingFunction { - UNDEFINED, - MAX, - AVG_EXCLUDE_PADDING, - AVG_INCLUDE_PADDING, -}; - -namespace Detail { - -class PoolingLayerFWDImpl; -class PoolingLayerBWDImpl; - -} // Detail - -class KDNN_API_PUBLIC PoolingLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const Propagation &propKind, const PoolingFunction &poolingAlg, - const TensorInfo &srcInfo, const TensorInfo &dstInfo, const Shape &strides, - const Shape &kernel, const Shape &dilation, const Shape &paddingL, const Shape &paddingR) noexcept; - PoolingLayerFWD(const Propagation &propKind, const PoolingFunction &poolingAlg, - const TensorInfo &srcInfo, const TensorInfo &dstInfo, const Shape &strides, - const Shape &kernel, const Shape &dilation, const Shape &paddingL, const Shape &paddingR) noexcept(false); - PoolingLayerFWD(const PoolingLayerFWD &other) noexcept(false); - PoolingLayerFWD(PoolingLayerFWD &&other) noexcept; - PoolingLayerFWD& operator=(const PoolingLayerFWD &other) noexcept(false); - PoolingLayerFWD& operator=(PoolingLayerFWD &&other) noexcept; - void Run(const void *src, void *dst, void *ws = nullptr) const noexcept(false); - void SetPadType(Element::Type newPadType) noexcept; - ~PoolingLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC PoolingLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const PoolingFunction &poolingAlg, - const TensorInfo &srcDiffInfo, const TensorInfo &dstDiffInfo, const Shape &strides, - const Shape &kernel, const Shape &dilation, const Shape &paddingL, const Shape &paddingR) noexcept; - PoolingLayerBWD(const PoolingFunction &poolingAlg, - const TensorInfo &srcInfo, const TensorInfo &dstInfo, const Shape &strides, - const Shape &kernel, const Shape &dilation, const Shape &paddingL, const Shape &paddingR) noexcept(false); - PoolingLayerBWD(const PoolingLayerBWD &other) noexcept(false); - PoolingLayerBWD(PoolingLayerBWD &&other) noexcept; - PoolingLayerBWD& operator=(const PoolingLayerBWD &other) noexcept(false); - PoolingLayerBWD& operator=(PoolingLayerBWD &&other) noexcept; - void Run(void *diffSrc, const void *diffDst, const void *ws = nullptr) const noexcept(false); - ~PoolingLayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_POOLING_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_prelu.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_prelu.hpp deleted file mode 100644 index bfff8678f9a73f33dc27eee18f53f2813fe31f29..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_prelu.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Prelu op - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_PRELU_HPP -#define KDNN_PRELU_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class PReLULayerFWDImpl; -class PReLULayerBWDImpl; - -} // Detail - -class KDNN_API_PUBLIC PReLULayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &weightsInfo, - const TensorInfo &dstInfo) noexcept; - PReLULayerFWD(const TensorInfo &srcInfo, const TensorInfo &weightsInfo, - const TensorInfo &dstInfo) noexcept(false); - PReLULayerFWD(const PReLULayerFWD &other) noexcept(false); - PReLULayerFWD(PReLULayerFWD &&other) noexcept; - PReLULayerFWD& operator=(const PReLULayerFWD &other) noexcept(false); - PReLULayerFWD& operator=(PReLULayerFWD &&other) noexcept; - void Run(const void *src, const void *wei, void *dst) const noexcept(false); - ~PReLULayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC PReLULayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &diffSrcInfo, - const TensorInfo &weightsInfo, const TensorInfo &diffWeightsInfo, - const TensorInfo &diffDstInfo) noexcept; - PReLULayerBWD(const TensorInfo &srcInfo, const TensorInfo &diffSrcInfo, - const TensorInfo &weightsInfo, const TensorInfo &diffWeightsInfo, - const TensorInfo &diffDstInfo) noexcept(false); - PReLULayerBWD(const PReLULayerBWD &other) noexcept(false); - PReLULayerBWD(PReLULayerBWD &&other) noexcept; - PReLULayerBWD& operator=(const PReLULayerBWD &other) noexcept(false); - PReLULayerBWD& operator=(PReLULayerBWD &&other) noexcept; - void Run(const void *src, void *diffSrc, const void *wei, - void *diffWeights, const void *diffDst) const noexcept(false); - ~PReLULayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_PRELU_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_reduction.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_reduction.hpp deleted file mode 100644 index bfba7e0d93204dfdd35bf9e69c938a7a5c14f074..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_reduction.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Reduction - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_REDUCTION_HPP -#define KDNN_REDUCTION_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -// supported functions -enum class ReductionFunction { - UNIMPLEMENTED, - MAX, - MIN, - SUM, - MUL, - MEAN, - NORM_LP_MAX, - NORM_LP_SUM, - NORM_LP_POWER_P_MAX, - NORM_LP_POWER_P_SUM, -}; - -namespace Detail { - -class ReductionLayerImpl; - -} // Detail - -class KDNN_API_PUBLIC ReductionLayer final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &dstInfo, - const ReductionFunction& reductionAlg, float p, float eps) noexcept; - ReductionLayer(const TensorInfo &srcInfo, const TensorInfo &dstInfo, ReductionFunction kind, - float p = 1, float eps = 0) noexcept(false); - ReductionLayer(const ReductionLayer &other) noexcept(false); - ReductionLayer(ReductionLayer &&other) noexcept; - ReductionLayer& operator=(const ReductionLayer &other) noexcept(false); - ReductionLayer& operator=(ReductionLayer &&other) noexcept; - void Run(const void *src, void *dst) const noexcept(false); - ~ReductionLayer() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_REDUCTION_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_reorder.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_reorder.hpp deleted file mode 100644 index d18acff066b022a42ed75af33d77f2279f1e23c4..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_reorder.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: reorder - * Author: KPL - * Create: 2024-06-25 - * Notes: NA - */ - -#ifndef KDNN_REORDER_HPP -#define KDNN_REORDER_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { -namespace Detail { - -class ReorderLayerImpl; - -} // Detail - -class KDNN_API_PUBLIC ReorderLayer final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &dstInfo) noexcept; - ReorderLayer(const TensorInfo &srcInfo, const TensorInfo &dstInfo) noexcept(false); - ReorderLayer(const ReorderLayer &other) noexcept(false); - ReorderLayer(ReorderLayer &&other) noexcept; - ReorderLayer& operator=(const ReorderLayer &other) noexcept(false); - ReorderLayer& operator=(ReorderLayer &&other) noexcept; - void Run(const void *src, void *dst) const noexcept(false); - ~ReorderLayer() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_REORDER_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_resampling.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_resampling.hpp deleted file mode 100644 index 1678100035e84bc59bca596e4da9a1d64a9432c2..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_resampling.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: mathematical correlation function - * Author: KPL - * Create: 2024-07-3 - * Notes: NA - */ - -#ifndef KDNN_RESAMPLING_HPP -#define KDNN_RESAMPLING_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -enum class ResamplingAlg { - UNIMPLEMENTED, - NEAREST_NEIGHBOR, - LINEAR, -}; - -namespace Detail { -namespace Resampling { - class ResamplingLayerImplFWD; - class ResamplingLayerImplBWD; -} // Resampling -} // Detail - -class KDNN_API_PUBLIC ResamplingLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &dstInfo, - ResamplingAlg algKind) noexcept; - ResamplingLayerFWD(const TensorInfo &srcInfo, const TensorInfo &dstInfo, - ResamplingAlg algKind) noexcept(false); - ResamplingLayerFWD(const ResamplingLayerFWD &other) noexcept(false); - ResamplingLayerFWD(ResamplingLayerFWD &&other) noexcept; - ResamplingLayerFWD& operator=(const ResamplingLayerFWD &other) noexcept(false); - ResamplingLayerFWD& operator=(ResamplingLayerFWD &&other) noexcept; - void Run(const void *src, void *dst) const noexcept(false); - ~ResamplingLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC ResamplingLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - ResamplingLayerBWD(const TensorInfo &diffDstInfo, const TensorInfo &diffSrcInfo, - ResamplingAlg algKind) noexcept(false); - - ResamplingLayerBWD(const ResamplingLayerBWD &other) noexcept(false); - ResamplingLayerBWD(ResamplingLayerBWD &&other) noexcept; - ResamplingLayerBWD& operator=(const ResamplingLayerBWD &other) noexcept(false); - ResamplingLayerBWD& operator=(ResamplingLayerBWD &&other) noexcept; - - ~ResamplingLayerBWD() noexcept; - - static Status ValidateInput(const TensorInfo &diffDstInfo, const TensorInfo &diffSrcInfo, - ResamplingAlg algKind) noexcept; - - void Run(const void *diffDst, void *diffSrc) const noexcept(false); - -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_rnn.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_rnn.hpp deleted file mode 100644 index 682edc97da7a319c0b79e188837ea964df742837..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_rnn.hpp +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. - * Description: Rnn - * Author: KPL - * Create: 2025-01-08 - * Notes: NA - */ - -#ifndef KDNN_RNN_HPP -#define KDNN_RNN_HPP - -#include -#include -#include "service/kdnn_err_codes.hpp" -#include "types/kdnn_tensor_info.hpp" - -namespace KDNN { - -enum class RnnAlgorithm { - UNIMPLEMENTED, - RNN, - LSTM, - GRU, - LBR_GRU, - AUGRU, - LBR_AUGRU -}; - -enum class ActivateFunctionRNN { - UNIMPLEMENTED, - RELU, - TANH, - LOGISTIC -}; - -namespace Detail { - -enum class ExecutionDirectionT { - L2R, - R2L, - BI_CONCAT, - BI_SUM, -}; - -enum class CellPositionT { - MIDDLE_CELL = 0x0, - FIRST_LAYER = 0x1, - FIRST_ITER = 0x2, - LAST_LAYER = 0x4, - LAST_ITER = 0x8, - C_STATE_FIRST_ITER = 0x10, - C_STATE_LAST_ITER = 0x20, - MERGED_ITER = 0x40, - MERGED_LAYER = 0x80 -}; - -enum class WeightsTypeT { - LAYER, - ITER, - PROJECTION, - PEEPHOLE, -}; - -inline CellPositionT &operator|=(CellPositionT &lhs, CellPositionT rhs) -{ - lhs = static_cast( - static_cast(lhs) | static_cast(rhs)); - return lhs; -} - -inline CellPositionT operator|(CellPositionT lhs, CellPositionT rhs) -{ - return static_cast( - static_cast(lhs) | static_cast(rhs)); -} - -inline bool operator&(CellPositionT lhs, CellPositionT rhs) -{ - return static_cast( - static_cast(lhs) & static_cast(rhs)); -} - -enum class BrgemmRnnExecuteLoopOrderT { - // default for kernels w/o loop order choice - UNDEFINED = 0x0, - // mBlocking loop is outermost - M_BLK_N_BLK = 0x1, - // nBlocking loop is outermost - N_BLK_M_BLK = 0x2 -}; - -struct DiffSrcBrgemmConfT { - SizeType m = 0, n = 0, k = 0; - - SizeType nBlock = 0, nBlocks = 0, nTail = 0; - SizeType mBlock = 0, mBlocks = 0; - - SizeType kBlocks = 0, kBlock = 0, kTail = 0; - SizeType kpadded = 0; - - SizeType nIter = 0, nLayer = 0; - SizeType nLayerBlocks = 0, nLayerTail = 0; - SizeType nIterBlocks = 0, nIterTail = 0; - SizeType lda = 0, ldb = 0, ldc = 0; - - BrgemmRnnExecuteLoopOrderT loopOrder - = BrgemmRnnExecuteLoopOrderT::UNDEFINED; - int gatesBlock; -}; - -struct DiffWeiBrgemmConfT { - SizeType m = 0, mLayer = 0, mIter = 0, n = 0, k = 0; - - SizeType nBlock = 0, nBlocks = 0, nTail = 0; - SizeType mBlock = 0, mBlocks = 0; - SizeType kBlocks = 0, kBlock = 0, kTail = 0; - SizeType kpadded = 0; - SizeType ldaLayer = 0, ldaIter = 0, ldb = 0, ldcIter = 0, ldcLayer = 0; - - bool globalTranspose = false; - - BrgemmRnnExecuteLoopOrderT loopOrder - = BrgemmRnnExecuteLoopOrderT::UNDEFINED; -}; - -constexpr int KDNN_RNN_MAX_N_PARTS = 4; - -namespace Rnn { -class RnnFWDImpl; -class RnnBWDImpl; -} // Rnn - -} // Detail - -struct RnnCfg { - using SizeType = KDNN::SizeType; - - KDNN::Detail::ExecutionDirectionT execDir; - - IntType nLayer = 0, nIter = 0, nDir = 0, nGates = 0; - IntType mb = 0; - IntType slc = 0, sic = 0, dhc = 0, dic = 0, dlc = 0; - - IntType nBias = 0; - - int weightsLayerLd = 0; - int diffWeightsLayerLd = 0; - int weightsIterLd = 0; - int diffWeightsIterLd = 0; - int weightsProjectionLd = 0; - int diffWeightsProjectionLd = 0, diffWeightsProjectionNld = 0; - - int wsGatesLd = 0, wsGatesNld = 0; - int wsHtLd = 0, wsHtNld = 0; - int projHtLd = 0; - int wsStatesLayerLd = 0, wsStatesLayerNld = 0; - int wsStatesIterLd = 0; - int wsDiffStatesLayerLd = 0; - int wsDiffStatesIterLd = 0; - int wsDiffStatesIterCLd = 0; - - int scratchGatesLd = 0, scratchGatesNld = 0; - int scratchDiffHtLd = 0; - - int srcLayerLd = 0, srcLayerCellLd = 0; - int srcIterLd = 0; - int srcIterCLd = 0; - int dstLayerLd = 0, dstLayerCellLd = 0; - int dstIterLd = 0; - int dstIterCLd = 0; - - bool isTraining = false, isLbr = false; - - SizeType wsGatesSize = 0; - SizeType wsHtSize = 0; - SizeType wsStatesLayerSize = 0; - SizeType wsStatesIterCSize = 0; - SizeType wsDiffStatesLayerSize = 0; - SizeType wsDiffStatesIterSize = 0; - SizeType wsDiffStatesIterCSize = 0; - SizeType scratchGatesSize = 0; - SizeType scratchCellSize = 0; - SizeType wsGridCompSize = 0; - SizeType wsPerCell = 0; - - bool srcLayerIsTrivialStride = false; - bool dstLayerIsTrivialStride = false; - - bool diffWeightsOverwrite = false; - - SizeType wsGatesOffset = 0; - SizeType wsHtOffset = 0; - SizeType wsStatesLayerOffset = 0; - SizeType wsStatesIterOffset = 0; - SizeType wsStatesIterCOffset = 0; - SizeType wsBiasOffset = 0; - SizeType wsDiffStatesLayerOffset = 0; - SizeType wsDiffStatesIterOffset = 0; - SizeType wsDiffStatesIterCOffset = 0; - SizeType wsGridCompOffset = 0; - SizeType scratchGatesOffset = 0; - SizeType scratchHtOffset = 0; - SizeType scratchDiffHtOffset = 0; - SizeType scratchCellOffset = 0; - - float alpha = 0; - inline bool SkipDstLayerCopy() const - { - return (execDir == KDNN::Detail::ExecutionDirectionT::L2R || execDir == KDNN::Detail::ExecutionDirectionT::R2L); - } - inline bool SkipDstIterCopy() const - { - return (execDir == KDNN::Detail::ExecutionDirectionT::L2R || - execDir == KDNN::Detail::ExecutionDirectionT::R2L) && (dstIterLd > 0); - } - - inline SizeType SrcLayerLd(KDNN::Detail::CellPositionT cellPosition) const - { - return (cellPosition & KDNN::Detail::CellPositionT::FIRST_LAYER) - ? srcLayerLd - : (cellPosition & KDNN::Detail::CellPositionT::LAST_ITER) - ? dstIterLd - : wsStatesLayerLd; - } - - inline SizeType SrcIterLd(KDNN::Detail::CellPositionT cellPosition) const - { - return (cellPosition & KDNN::Detail::CellPositionT::FIRST_ITER) - ? srcIterLd - : ((cellPosition & KDNN::Detail::CellPositionT::LAST_LAYER) && SkipDstLayerCopy() - && !(cellPosition & KDNN::Detail::CellPositionT::FIRST_ITER) - ? dstLayerLd - : wsStatesIterLd); - } - - inline SizeType SrcIterCLd(KDNN::Detail::CellPositionT cellPosition) const - { - return isTraining ? - ((cellPosition & KDNN::Detail::CellPositionT::FIRST_ITER) ? srcIterCLd : wsDiffStatesIterCLd) : - (cellPosition & KDNN::Detail::CellPositionT::FIRST_ITER) ? srcIterCLd : dstIterCLd; - } - inline SizeType DstIterCLd(KDNN::Detail::CellPositionT cellPosition) const - { - return isTraining ? - ((cellPosition & KDNN::Detail::CellPositionT::LAST_ITER) ? dstIterCLd : wsDiffStatesIterCLd) : - dstIterCLd; - } - inline SizeType DstLayerLd(KDNN::Detail::CellPositionT cellPosition) const - { - return (cellPosition & KDNN::Detail::CellPositionT::LAST_LAYER) && SkipDstLayerCopy() - ? dstLayerLd - : (cellPosition & KDNN::Detail::CellPositionT::LAST_ITER) && SkipDstIterCopy() - ? dstIterLd - : wsStatesLayerLd; - } - - inline SizeType DstIterLd(KDNN::Detail::CellPositionT cellPosition) const - { - return (cellPosition & KDNN::Detail::CellPositionT::LAST_ITER) && SkipDstLayerCopy() - ? dstIterLd - : wsStatesLayerLd; - } - - inline SizeType DstIterPart2Ld(KDNN::Detail::CellPositionT cellPosition) const - { - return (cellPosition & KDNN::Detail::CellPositionT::LAST_LAYER) ? DstLayerLd(cellPosition) - : DstIterLd(cellPosition); - } - - // get DiffWeightsBeta based on cell position - inline float DiffWeightsBeta(KDNN::Detail::CellPositionT cellPosition) const - { - if (diffWeightsOverwrite && (cellPosition & KDNN::Detail::CellPositionT::LAST_ITER)) { - // Initialize diff weights if needed - return 0.0f; - } - return 1.0f; - } -}; - -class KDNN_API_PUBLIC RnnFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const std::vector &srcInfos, const std::vector &weightInfos, - const std::vector &dstInfos, TensorInfo &biasInfo, RnnCfg &rnn, - RnnAlgorithm alg, const ActivateFunctionRNN actf) noexcept; - RnnFWD(const std::vector &srcInfos, const std::vector &weightInfos, - const std::vector &dstInfos, TensorInfo &biasInfo, RnnCfg &rnn, RnnAlgorithm alg, - const ActivateFunctionRNN actf) noexcept(false); - RnnFWD(const RnnFWD &other) noexcept(false); - RnnFWD(RnnFWD &&other) noexcept; - RnnFWD& operator=(const RnnFWD &other) noexcept(false); - RnnFWD& operator=(RnnFWD &&other) noexcept; - void Run(const void **src, const void **weight, void **dst, const void *bias, void *wsBase, - const void *augruAttentions) const noexcept(false); - ~RnnFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC RnnBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const std::vector &srcInfos, const std::vector &weightInfos, - const std::vector &dstInfos, TensorInfo &biasInfo, - const std::vector &diffSrcInfos, const std::vector &diffWeightInfos, - const std::vector &diffDstInfos, TensorInfo &diffBiasInfo, RnnAlgorithm alg, - const ActivateFunctionRNN actf) noexcept; - RnnBWD(const std::vector &srcInfos, const std::vector &weightInfos, - const std::vector &dstInfos, TensorInfo &biasInfo, - const std::vector &diffSrcInfos, const std::vector &diffWeightInfos, - const std::vector &diffDstInfos, TensorInfo &diffBiasInfo, RnnCfg &rnn, RnnAlgorithm alg, - const ActivateFunctionRNN actf) noexcept(false); - RnnBWD(const RnnBWD &other) noexcept(false); - RnnBWD(RnnBWD &&other) noexcept; - RnnBWD& operator=(const RnnBWD &other) noexcept(false); - RnnBWD& operator=(RnnBWD &&other) noexcept; - void Run(const void **src, const void **weight, const void **dst, const void *bias, void *wsBase, - const void **diffDstVec, void **diffSrcVec, void **diffWeightVec, void *diffBias, - const void *augruAttention, void *diffAugruAttention) const noexcept(false); - ~RnnBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_RNN_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_shuffle.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_shuffle.hpp deleted file mode 100644 index b7131591ae9a4ddb5e49d3e9ce08e2ffffa8c666..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_shuffle.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: shuffle correlation function - * Author: KPL - * Create: 2024-07-20 - * Notes: NA - */ - -#ifndef KDNN_SHUFFLE_HPP -#define KDNN_SHUFFLE_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class ShuffleLayerImpl; - -} // Detail - -class KDNN_API_PUBLIC ShuffleLayer final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &dstInfo, SizeType axis, - SizeType groupSize) noexcept; - ShuffleLayer(const TensorInfo &srcInfo, const TensorInfo &dstInfo, SizeType axis, - SizeType group) noexcept(false); - ShuffleLayer(const ShuffleLayer &other) noexcept(false); - ShuffleLayer(ShuffleLayer &&other) noexcept; - ShuffleLayer& operator=(const ShuffleLayer &other) noexcept(false); - ShuffleLayer& operator=(ShuffleLayer &&other) noexcept; - void Run(const void *src, void *dst) const noexcept(false); - ~ShuffleLayer() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_SHUFFLE_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_softmax.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_softmax.hpp deleted file mode 100644 index 2adea5bbfe0a6da6f47a5b1642aa09966a0f9348..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_softmax.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Softmax - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_SOFTMAX_HPP -#define KDNN_SOFTMAX_HPP - -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -enum class SoftmaxAlgorithmKind : std::uint32_t { - SOFTMAX = 0x0U, - LOGSOFTMAX = 0x1U, -}; - -namespace Detail { - -class SoftmaxLayerFWDImpl; -class SoftmaxLayerBWDImpl; - -} // Detail - -class KDNN_API_PUBLIC SoftmaxLayerFWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &srcInfo, const TensorInfo &dstInfo, SizeType axis, - SoftmaxAlgorithmKind algorithm) noexcept; - SoftmaxLayerFWD(const TensorInfo &srcInfo, const TensorInfo &dstInfo, SizeType axis, - SoftmaxAlgorithmKind algorithm) noexcept(false); - SoftmaxLayerFWD(const SoftmaxLayerFWD &other) noexcept(false); - SoftmaxLayerFWD(SoftmaxLayerFWD &&other) noexcept; - SoftmaxLayerFWD& operator=(const SoftmaxLayerFWD &other) noexcept(false); - SoftmaxLayerFWD& operator=(SoftmaxLayerFWD &&other) noexcept; - void Run(const void *src, void *dst) const noexcept(false); - ~SoftmaxLayerFWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -class KDNN_API_PUBLIC SoftmaxLayerBWD final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const TensorInfo &dstInfo, const TensorInfo &dstDiffInfo, - const TensorInfo &srcDiffInfo, SizeType axis, SoftmaxAlgorithmKind algorithm) noexcept; - SoftmaxLayerBWD(const TensorInfo &dstInfo, const TensorInfo &dstDiffInfo, const TensorInfo &srcDiffInfo, - SizeType axis, SoftmaxAlgorithmKind algorithm) noexcept(false); - SoftmaxLayerBWD(const SoftmaxLayerBWD &other) noexcept(false); - SoftmaxLayerBWD(SoftmaxLayerBWD &&other) noexcept; - SoftmaxLayerBWD& operator=(const SoftmaxLayerBWD &other) noexcept(false); - SoftmaxLayerBWD& operator=(SoftmaxLayerBWD &&other) noexcept; - void Run(const void *dst, const void *dstDiff, void *srcDiff) const noexcept(false); - ~SoftmaxLayerBWD() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_SOFTMAX_HPP diff --git a/third_party/KDNN/kdnn_include/operations/kdnn_sum.hpp b/third_party/KDNN/kdnn_include/operations/kdnn_sum.hpp deleted file mode 100644 index c8a80c9fad1a69698eff08ad2b73abbd3f6d67ec..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/operations/kdnn_sum.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Sum - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_SUM_HPP -#define KDNN_SUM_HPP - -#include -#include - -#include "types/kdnn_tensor_info.hpp" -#include "service/kdnn_err_codes.hpp" - -namespace KDNN { - -namespace Detail { - -class SumLayerImpl; - -} // Detail - -class KDNN_API_PUBLIC SumLayer final { -public: - using SizeType = ::KDNN::SizeType; - static Status ValidateInput(const std::vector &srcInfo, - const float *scales, const TensorInfo &dstInfo) noexcept; - SumLayer(const std::vector &srcInfo, const float *scales, - const TensorInfo &dstInfo) noexcept(false); - SumLayer(const SumLayer &other) noexcept(false); - SumLayer(SumLayer &&other) noexcept; - SumLayer& operator=(const SumLayer &other) noexcept(false); - SumLayer& operator=(SumLayer &&other) noexcept; - void Run(const void **src, void *dst) const noexcept(false); - ~SumLayer() noexcept; -private: - std::unique_ptr pImpl; -}; - -} // KDNN - -#endif // KDNN_SUM_HPP diff --git a/third_party/KDNN/kdnn_include/service/kdnn_err_codes.hpp b/third_party/KDNN/kdnn_include/service/kdnn_err_codes.hpp deleted file mode 100644 index 4f9ecd0b726981975db10c8b3043b0f42d7d8195..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/service/kdnn_err_codes.hpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Error codes - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_ERR_CODES -#define KDNN_ERR_CODES - -namespace KDNN { - -enum class Status { - SUCCESS = 0, - UNSUPPORTED, - BAD_ARGUMENTS -}; - -} // KDNN - -#endif // KDNN_ERR_CODES diff --git a/third_party/KDNN/kdnn_include/service/kdnn_exception.hpp b/third_party/KDNN/kdnn_include/service/kdnn_exception.hpp deleted file mode 100644 index 977dc59d43871f329fb75799e37976f1896c1cd8..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/service/kdnn_exception.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Exceptions - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_EXCEPTION_HPP -#define KDNN_EXCEPTION_HPP - -#include -#include -#include - -namespace KDNN { -namespace Service { - -// Error handling in KDNN is performed via exceptions. -// KDNN has its own exceptions which are nested from std::exception standard class -struct BadAlloc : public std::bad_alloc { - const char* what() const noexcept override - { - return "KDNN allocation failed"; - } -}; - -struct BadArrayNewLength : public std::bad_array_new_length { - const char* what() const noexcept override - { - return "KDNN bad array new length"; - } -}; - -struct LogicError : public std::logic_error { - explicit LogicError(const std::string& whatArg) : std::logic_error(whatArg) {} - explicit LogicError(const char* whatArg) : std::logic_error(whatArg) {} - LogicError(const LogicError& other) noexcept : std::logic_error(other) {} - LogicError& operator=(const LogicError& other) noexcept = default; -}; - -struct Unsupported : public std::invalid_argument { - explicit Unsupported(const std::string& whatArg) : std::invalid_argument(whatArg) {} - explicit Unsupported(const char* whatArg) : std::invalid_argument(whatArg) {} - Unsupported(const Unsupported& other) noexcept : std::invalid_argument(other) {} - Unsupported& operator=(const Unsupported& other) noexcept = default; -}; - -} // Service -} // KDNN - -#endif // KDNN_EXCEPTION_HPP diff --git a/third_party/KDNN/kdnn_include/service/kdnn_service.hpp b/third_party/KDNN/kdnn_include/service/kdnn_service.hpp deleted file mode 100644 index 5d825c5c290aec513c57c13d11fe8029398d0b7f..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/service/kdnn_service.hpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Allocator - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_SERVICE_HPP -#define KDNN_SERVICE_HPP - -#include -#include -#include - -#include "types/kdnn_types.hpp" -#include "types/kdnn_data_type.hpp" -#include "types/kdnn_layout.hpp" -#include "service/kdnn_exception.hpp" - -namespace KDNN { -namespace Service { - -constexpr SizeType ALIGNMENT = 128; - -KDNN_API_PUBLIC SizeType GetIdxForUserLayout(Layout l, SizeType idx) noexcept(false); - -KDNN_API_PUBLIC Shape GetShapeAccordingToLayout(const Shape &sh, Layout parLayout) noexcept; - -KDNN_API_PUBLIC Shape FlushStrides(const Shape &dims, const Shape &strides) noexcept(false); - -template -struct IsAllSame : std::false_type {}; - -template<> -struct IsAllSame<> : std::true_type {}; - -template -struct IsAllSame : std::true_type {}; - -template -struct IsAllSame : std::true_type {}; - -template -struct IsAllSame : IsAllSame {}; - -template ::type>::value, - bool>::type = true> -bool WillIntMultOverflow(IntType a, IntType b) noexcept -{ - if ((a == 0) || (b == 0)) { - return false; - } else { - volatile IntType mul = a * b; - return (mul / a) != b; - } -} - -template ::value, bool>::type = true> -bool WillIntMultOverflow(IntType a, IntType b, Args ... args) noexcept -{ - if (WillIntMultOverflow(a, b)) { - return true; - } else { - return WillIntMultOverflow(a * b, args ...); - } -} - -template ::value_type>::value, - bool>::type = true> -bool WillIntMultOverflow(ForwardIt begin, ForwardIt end) noexcept -{ - typename std::iterator_traits::value_type mult = 1; - for (auto &&it = begin; it != end; ++it) { - if (WillIntMultOverflow(mult, *it)) { - return true; - } - mult *= *it; - } - return false; -} - -// User may use allocation/deallocation functions directly or via allocator class -KDNN_API_PUBLIC void *AlignedAlloc(SizeType n, SizeType alignment = ALIGNMENT) noexcept(false); -KDNN_API_PUBLIC void Deallocate(void *p, SizeType n = 0) noexcept; - -// This class represents an allocator which user may use in order to obtain aligned memory -// and pass it inside KDNN functions. To be compatible with std::allocator all member names are in snake_case -template -struct AlignedAllocator { - using value_type = T; - using pointer = T *; - using const_pointer = const T *; - using reference = T &; - using const_reference = const T &; - using size_type = ::KDNN::SizeType; - using difference_type = std::ptrdiff_t; - - template struct rebind { - using other = AlignedAllocator; - }; - - T *allocate(SizeType n) const noexcept(false) - { - if (n > std::numeric_limits::max() / sizeof(T)) { - throw BadArrayNewLength(); - } - return static_cast(AlignedAlloc(n * sizeof(T), alignment)); - } - - void deallocate(T *p, SizeType n = 0) const noexcept - { - ::KDNN::Service::Deallocate(p, n); - } - virtual ~AlignedAllocator() = default; -}; - -template -struct Deallocator { - void operator()(T *p, ::KDNN::SizeType n = 0) const noexcept - { - ::KDNN::Service::Deallocate(p, n); - } -}; - -template -constexpr bool operator == (const AlignedAllocator &, - const AlignedAllocator &) noexcept -{ - return (alignment_1 == alignment_2) && std::is_same::value; -} -template -constexpr bool operator != (const AlignedAllocator &lhs, - const AlignedAllocator &rhs) noexcept -{ - return !(lhs == rhs); -} - -} // Service -} // KDNN - -#endif // KDNN_SERVICE_HPP diff --git a/third_party/KDNN/kdnn_include/service/kdnn_threading.hpp b/third_party/KDNN/kdnn_include/service/kdnn_threading.hpp deleted file mode 100644 index 4aa4d9ff5e983675a20a0519debdc929e123da2c..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/service/kdnn_threading.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Error codes - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_THREADING_HPP -#define KDNN_THREADING_HPP - -#include -#include - -namespace KDNN { -namespace Threading { - -struct ThreadpoolWrapper { - /// Returns the number of worker threads. - virtual int GetNumThreads() const = 0; - - /// Returns true if the calling thread belongs to this threadpool. - virtual bool GetInParallel() const = 0; - - virtual void Parallel(int n, const std::function &fn) = 0; - - virtual ~ThreadpoolWrapper() {} -}; - -enum class EnvMode { - KDNN_THREAD_USE_ENV, - KDNN_THREAD_IGNORE_ENV -}; - -enum class ThreadingControl { - KDNN_DEFAULT, - KDNN_MANUAL -}; - -KDNN_API_PUBLIC bool ShouldCalculateOptThreads() noexcept; -KDNN_API_PUBLIC int SetMaxNumThreads(int nThreads) noexcept; -KDNN_API_PUBLIC int SetFixedNumThreads(int nThreads) noexcept; -KDNN_API_PUBLIC int GetMaxNumThreads() noexcept; -KDNN_API_PUBLIC int GetMaxNumThreads(void* thread_pool) noexcept; -KDNN_API_PUBLIC int SetNumThreadsLocal(int nThreads) noexcept; -KDNN_API_PUBLIC EnvMode SetEnvMode(EnvMode mode) noexcept; -KDNN_API_PUBLIC ThreadingControl GetThreadingControlStatus() noexcept; - -} // Threading -} // KDNN - -#endif // KDNN_THREADING_HPP diff --git a/third_party/KDNN/kdnn_include/types/kdnn_data_type.hpp b/third_party/KDNN/kdnn_include/types/kdnn_data_type.hpp deleted file mode 100644 index 65c763716720eec2cc223ed45450acf1b59a26e7..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/types/kdnn_data_type.hpp +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Provide runtime info about data type. - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_DATATYPE_HPP -#define KDNN_DATATYPE_HPP - -#include -#include - -#include "service/kdnn_exception.hpp" - -#define KDNN_API_PUBLIC __attribute__((visibility("default"))) - -#define GCC_VER (__GNUC__ * 100 + __GNUC_MINOR__) - -#if GCC_VER >= 1000 -typedef __fp16 fp16_t; -#else -struct fp16_t { - uint16_t data; -}; -#endif - -#if GCC_VER >= 1000 -typedef __bf16 bf16_t; -#else -struct bf16_t { - uint16_t data; -}; -#endif - -namespace KDNN { - -using SizeType = std::size_t; -using IntType = std::int64_t; - -namespace Element { - -// enumeration of KDNN supported data types -enum class TypeT { - UNDEFINED = 0, - F32, - F16, - BF16, - S32, - S8, - U8 -}; - -// This class is designed to provide runtime info about data type. -struct Type final { -public: - using SizeType = ::KDNN::SizeType; - Type(const TypeT& t) noexcept(false) : type{t}, size(0) - { - switch (type) { - case TypeT::F32: { - size = sizeof(float); - break; - } - case TypeT::F16: { - size = sizeof(fp16_t); - break; - } - case TypeT::BF16: { - size = sizeof(bf16_t); - break; - } - case TypeT::S32: { - size = sizeof(std::int32_t); - break; - } - case TypeT::S8: { - size = sizeof(std::int8_t); - break; - } - case TypeT::U8: { - size = sizeof(std::uint8_t); - break; - } - default: {} - } - if (type == TypeT::UNDEFINED) { - // throw Service::LogicError {"Type: unsupported data type"}; - size = 0; - } - } - SizeType GetSize() const noexcept - { - return size; - } - SizeType GetBitwidth() const noexcept - { - return (GetSize() * 8uLL); - } - operator TypeT() const noexcept - { - return type; - } - bool IsFP() const noexcept - { - if ((type == TypeT::F32) || - (type == TypeT::F16) || - (type == TypeT::BF16)) { - return true; - } else { - return false; - } - } - bool IsIntegral() const noexcept - { - return !IsFP(); - } - bool IsSigned() const noexcept - { - if (type == TypeT::U8) { - return false; - } else { - return true; - } - } - bool IsUnsigned() const noexcept - { - return !IsSigned(); - } -private: - TypeT type; - SizeType size; -}; - -template -inline Type MatchType() -{ - return TypeT::UNDEFINED; -} - -template <> -inline Type MatchType() -{ - return TypeT::F32; -} - -template <> -inline Type MatchType() -{ - return TypeT::F16; -} - -template <> -inline Type MatchType() -{ - return TypeT::BF16; -} - -template <> -inline Type MatchType() -{ - return TypeT::S32; -} - -template <> -inline Type MatchType() -{ - return TypeT::S8; -} - -template <> -inline Type MatchType() -{ - return TypeT::U8; -} - -inline bool operator==(const Type &lhs, const TypeT &rhs) noexcept -{ - return (static_cast(lhs) == rhs); -} - -inline bool operator==(const TypeT &lhs, const Type &rhs) noexcept -{ - return (lhs == static_cast(rhs)); -} - -inline bool operator==(const Type &lhs, const Type &rhs) noexcept -{ - return (lhs.GetSize() == rhs.GetSize()) && - (static_cast(lhs) == (static_cast(rhs))); -} - -inline bool operator!=(const Type &lhs, const TypeT &rhs) noexcept -{ - return !(lhs == rhs); -} -inline bool operator!=(const TypeT &lhs, const Type &rhs) noexcept -{ - return !(lhs == rhs); -} -inline bool operator!=(const Type &lhs, const Type &rhs) noexcept -{ - return !(lhs == rhs); -} - -} // Element -} // KDNN - -#endif // KDNN_DATATYPE_HPP diff --git a/third_party/KDNN/kdnn_include/types/kdnn_layout.hpp b/third_party/KDNN/kdnn_include/types/kdnn_layout.hpp deleted file mode 100644 index 357630a46ef5aa33f2eb954e534ae8ca47d9003a..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/types/kdnn_layout.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Enumeration of common data types which are supported in KDNN. - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_LAYOUT_HPP -#define KDNN_LAYOUT_HPP - -namespace KDNN { - -// Enumeration of common data types which are supported in KDNN. -enum class Layout { - UNDEFINED = 0, - A, // single row - AB, - BA, - ABC, - ACB, - BAC, - BCA, - CAB, - CBA, - ABCD, - ABDC, - ACBD, - ACDB, - ADBC, - ADCB, - BACD, - BCDA, - CDAB, - CDBA, - DCAB, - ABCDE, - ABCED, - ABDEC, - ACBDE, - ACDEB, - ADECB, - BACDE, - BCDEA, - CDEAB, - CDEBA, - DECAB, - ROW_MAJOR = AB, - COL_MAJOR = BA, - NCHW = ABCD, - NHWC = ACDB, - NCDHW = ABCDE, - NDHWC = ACDEB, - OIHW = ABCD, - HWIO = CDBA, - HWOI = CDAB, - OHWI = ACDB, - OHWO = BCDA, - IOHW = BACD, -}; - -} // KDNN - -#endif // KDNN_LAYOUT_HPP diff --git a/third_party/KDNN/kdnn_include/types/kdnn_normalization.hpp b/third_party/KDNN/kdnn_include/types/kdnn_normalization.hpp deleted file mode 100644 index 71cc3ba7e452b649f98022330e288ab77b3bf360..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/types/kdnn_normalization.hpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Enumeration of common data types which are supported in KDNN. - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_NORMALIZATION_HPP -#define KDNN_NORMALIZATION_HPP - -#include "types/kdnn_data_type.hpp" - -namespace KDNN { - -enum class KDNN_API_PUBLIC NormalizationFlags : std::uint32_t { - NONE = 0x0U, - USE_GLOBAL_STATS = 0x1U, - USE_SCALE = 0x2U, - USE_SHIFT = 0x4U, - FUSE_NORM_RELU = 0x8U -}; - -KDNN_API_PUBLIC NormalizationFlags operator & (const NormalizationFlags &lhs, const NormalizationFlags &rhs); -KDNN_API_PUBLIC NormalizationFlags operator | (const NormalizationFlags &lhs, const NormalizationFlags &rhs); -KDNN_API_PUBLIC NormalizationFlags &operator &= (NormalizationFlags &lhs, const NormalizationFlags &rhs); -KDNN_API_PUBLIC NormalizationFlags &operator |= (NormalizationFlags &lhs, const NormalizationFlags &rhs); - -} // namespace KDNN - -#endif // KDNN_NORMALIZATION_HPP diff --git a/third_party/KDNN/kdnn_include/types/kdnn_propagation_type.hpp b/third_party/KDNN/kdnn_include/types/kdnn_propagation_type.hpp deleted file mode 100644 index 03cb05d5f33d6937702c33bbce413a4236d5a922..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/types/kdnn_propagation_type.hpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Enumeration of common propagation types which are supported in KDNN. - * Author: KPL - * Create: 2024-07-19 - * Notes: NA - */ - -#ifndef KDNN_PROPAGATION_TYPE_HPP -#define KDNN_PROPAGATION_TYPE_HPP - -namespace KDNN { - -// Enumeration of common propagation types which are supported in KDNN. -enum class Propagation { - UNDEFINED = 0, - FORWARD_TRAINING, - FORWARD_INFERENCE, - BACKWARD_DATA, - BACKWARD_WEIGHTS, - BACKWARD_BIAS, - BACKWARD, - FORWARD = FORWARD_TRAINING, -}; - -} // KDNN - -#endif // KDNN_PROPAGATION_TYPE_HPP diff --git a/third_party/KDNN/kdnn_include/types/kdnn_shape.hpp b/third_party/KDNN/kdnn_include/types/kdnn_shape.hpp deleted file mode 100644 index 3745f4dc4546c275f6ba89534e8afa1185fdf059..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/types/kdnn_shape.hpp +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Shape class designed to specify multidimensional data shape. - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_SHAPE_HPP -#define KDNN_SHAPE_HPP - -#include -#include - -#include "service/kdnn_service.hpp" -#include "types/kdnn_types.hpp" -#include "service/kdnn_exception.hpp" - -namespace KDNN { - -constexpr SizeType MAX_DIMS = 5; - -// This class designed to specify multidimensional data shape -struct Shape final { - using SizeType = ::KDNN::SizeType; - static constexpr SizeType NUM_MAX_DIMENSIONS = MAX_DIMS; - - Shape() noexcept : dimsArray{}, numDims{0} - { - for (SizeType i = 0; i < NUM_MAX_DIMENSIONS; ++i) { - dimsArray[i] = 0; - } - } - - template - Shape(T *ptr, const SizeType size) noexcept(false) : numDims(size) - { - if (ptr == nullptr) { - // throw Service::LogicError("Shape: ptr is nullptr"); - } - CheckNumDims(numDims); - for (SizeType i = 0; i < numDims; ++i) { - dimsArray[i] = ptr[i]; - } - for (SizeType i = numDims; i < NUM_MAX_DIMENSIONS; ++i) { - dimsArray[i] = 0; - } - } - template - Shape(Ts... dims) noexcept(false) : dimsArray{{static_cast(dims)...}}, numDims{sizeof...(dims)} - { - CheckNumDims(numDims); - } - Shape(const std::initializer_list& list) noexcept(false) - { - ResetShape(list); - } - - template - Shape& ResetShape(Ts... dims) noexcept(false) - { - CheckNumDims(sizeof...(dims)); - numDims = sizeof...(dims); - dimsArray = {static_cast(dims)...}; - for (SizeType i = numDims; i < NUM_MAX_DIMENSIONS; ++i) { - dimsArray[i] = 0; - } - return *this; - } - Shape& ResetShape(const std::initializer_list& list) noexcept(false) - { - CheckNumDims(list.size()); - numDims = list.size(); - for (SizeType i = 0; i < numDims; ++i) { - dimsArray[i] = *(list.begin() + i); - } - for (SizeType i = numDims; i < NUM_MAX_DIMENSIONS; ++i) { - dimsArray[i] = 0; - } - return *this; - } - template - Shape& ResetShape(T *ptr, const SizeType size) noexcept(false) - { - if (ptr == nullptr) { - // throw Service::LogicError("Shape: ptr is nullptr"); - } - CheckNumDims(size); - numDims = size; - for (SizeType i = 0; i < numDims; ++i) { - dimsArray[i] = ptr[i]; - } - for (SizeType i = numDims; i < NUM_MAX_DIMENSIONS; ++i) { - dimsArray[i] = 0; - } - return *this; - } - - Shape& operator+=(const Shape &adder) noexcept(false) - { - if (adder.GetNumDims() != this->GetNumDims()) { - // throw Service::LogicError("Shape: different size of base and adder shapes"); - } - for (SizeType i = 0; i < adder.GetNumDims(); ++i) { - this->operator[](i) += adder[i]; - } - return *this; - } - - Shape operator+(const Shape &adder) noexcept(false) - { - if (adder.GetNumDims() != this->GetNumDims()) { - // throw Service::LogicError("Shape: different size of base and adder shapes"); - } - std::array tmp; - for (SizeType i = 0; i < adder.GetNumDims(); ++i) { - tmp[i] = this->operator[](i) + adder[i]; - } - return Shape(tmp.data(), this->GetNumDims()); - } - - SizeType operator[](SizeType id) const noexcept(false) - { - if (id >= numDims) { - // throw Service::LogicError("Shape: index >= num_dims"); - } - return dimsArray[id]; - } - - SizeType& operator[](SizeType id) noexcept(false) - { - if (id >= numDims) { - // throw Service::LogicError("Shape: index >= num_dims"); - } - return dimsArray[id]; - } - - SizeType GetNumDims() const noexcept - { - return numDims; - } - - SizeType GetTotalDimsSize() const noexcept(false) - { - if (Service::WillIntMultOverflow(dimsArray.begin(), dimsArray.begin() + numDims)) { - // throw Service::LogicError("Shape: computing total size will cause overflow"); - } - SizeType accum = 1; - for (SizeType i = 0; i < numDims; ++i) { - accum *= dimsArray[i]; - } - return accum; - } - -private: - void CheckNumDims(SizeType nDims) const noexcept(false) - { - if (nDims > NUM_MAX_DIMENSIONS) { - // throw Service::LogicError("Shape: dims is greater than NUM_MAX_DIMENSIONS"); - } - } - std::array dimsArray; - SizeType numDims; -}; - -inline bool operator==(const Shape &lhs, const Shape &rhs) noexcept -{ - if (lhs.GetNumDims() == rhs.GetNumDims()) { - for (SizeType i = 0; i < lhs.GetNumDims(); ++i) { - if (lhs[i] != rhs[i]) { - return false; - } - } - return true; - } - return false; -} - -inline bool operator!=(const Shape &lhs, const Shape &rhs) noexcept -{ - return !(lhs == rhs); -} - -} // KDNN - -#endif // KDNN_SHAPE_HPP diff --git a/third_party/KDNN/kdnn_include/types/kdnn_tensor_info.hpp b/third_party/KDNN/kdnn_include/types/kdnn_tensor_info.hpp deleted file mode 100644 index e065273798faf701be31b8262473513f12f353be..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/types/kdnn_tensor_info.hpp +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Tensor_info class is designed to describe multidimensional data: it's type, layout, shape and strides - * Author: KPL - * Create: 2024-04-10 - * Notes: NA - */ - -#ifndef KDNN_TENSORINFO_HPP -#define KDNN_TENSORINFO_HPP - -#include "types/kdnn_types.hpp" -#include "types/kdnn_data_type.hpp" -#include "types/kdnn_layout.hpp" -#include "types/kdnn_shape.hpp" -#include "service/kdnn_service.hpp" - -namespace KDNN { - -constexpr SizeType DIM_1 = 1; -constexpr SizeType DIM_2 = 2; -constexpr SizeType DIM_3 = 3; -constexpr SizeType DIM_4 = 4; -constexpr SizeType DIM_5 = 5; - -constexpr SizeType IDX_0 = 0; -constexpr SizeType IDX_1 = 1; -constexpr SizeType IDX_2 = 2; -constexpr SizeType IDX_3 = 3; -constexpr SizeType IDX_4 = 4; -constexpr SizeType IDX_5 = 5; - -constexpr IntType NUM_0 = 0; -constexpr IntType NUM_1 = 1; -constexpr IntType NUM_2 = 2; -constexpr IntType NUM_3 = 3; -constexpr IntType NUM_4 = 4; -constexpr IntType NUM_5 = 5; -constexpr IntType NUM_6 = 6; -constexpr IntType NUM_7 = 7; -constexpr IntType NUM_8 = 8; -constexpr IntType NUM_9 = 9; -constexpr IntType NUM_10 = 10; -constexpr IntType NUM_11 = 11; -constexpr IntType NUM_12 = 12; -constexpr IntType NUM_13 = 13; -constexpr IntType NUM_14 = 14; -constexpr IntType NUM_15 = 15; -constexpr IntType NUM_16 = 16; -constexpr IntType NUM_17 = 17; -constexpr IntType NUM_18 = 18; -constexpr IntType NUM_19 = 19; -constexpr IntType NUM_20 = 20; -constexpr IntType NUM_21 = 21; -constexpr IntType NUM_22 = 22; -constexpr IntType NUM_23 = 23; -constexpr IntType NUM_24 = 24; -constexpr IntType NUM_25 = 25; -constexpr IntType NUM_26 = 26; -constexpr IntType NUM_27 = 27; -constexpr IntType NUM_28 = 28; -constexpr IntType NUM_29 = 29; -constexpr IntType NUM_30 = 30; -constexpr IntType NUM_31 = 31; -constexpr IntType NUM_32 = 32; -constexpr IntType NUM_33 = 33; -constexpr IntType NUM_34 = 34; -constexpr IntType NUM_35 = 35; -constexpr IntType NUM_36 = 36; -constexpr IntType NUM_37 = 37; -constexpr IntType NUM_38 = 38; -constexpr IntType NUM_39 = 39; -constexpr IntType NUM_40 = 40; -constexpr IntType NUM_41 = 41; -constexpr IntType NUM_42 = 42; -constexpr IntType NUM_43 = 43; -constexpr IntType NUM_44 = 44; -constexpr IntType NUM_45 = 45; -constexpr IntType NUM_46 = 46; -constexpr IntType NUM_47 = 47; -constexpr IntType NUM_48 = 48; -constexpr IntType NUM_49 = 49; -constexpr IntType NUM_50 = 50; -constexpr IntType NUM_51 = 51; -constexpr IntType NUM_52 = 52; -constexpr IntType NUM_53 = 53; -constexpr IntType NUM_54 = 54; -constexpr IntType NUM_55 = 55; -constexpr IntType NUM_56 = 56; -constexpr IntType NUM_57 = 57; -constexpr IntType NUM_58 = 58; -constexpr IntType NUM_59 = 59; -constexpr IntType NUM_60 = 60; -constexpr IntType NUM_61 = 61; -constexpr IntType NUM_62 = 62; -constexpr IntType NUM_63 = 63; -constexpr IntType NUM_64 = 64; -constexpr IntType NUM_65 = 65; -constexpr IntType NUM_66 = 66; -constexpr IntType NUM_67 = 67; -constexpr IntType NUM_68 = 68; -constexpr IntType NUM_69 = 69; -constexpr IntType NUM_70 = 70; -constexpr IntType NUM_71 = 71; -constexpr IntType NUM_72 = 72; -constexpr IntType NUM_73 = 73; -constexpr IntType NUM_74 = 74; -constexpr IntType NUM_75 = 75; -constexpr IntType NUM_76 = 76; -constexpr IntType NUM_77 = 77; -constexpr IntType NUM_78 = 78; -constexpr IntType NUM_79 = 79; -constexpr IntType NUM_80 = 80; -constexpr IntType NUM_81 = 81; -constexpr IntType NUM_82 = 82; -constexpr IntType NUM_83 = 83; -constexpr IntType NUM_84 = 84; -constexpr IntType NUM_85 = 85; -constexpr IntType NUM_86 = 86; -constexpr IntType NUM_87 = 87; -constexpr IntType NUM_88 = 88; -constexpr IntType NUM_89 = 89; -constexpr IntType NUM_120 = 120; -constexpr IntType NUM_132 = 132; -constexpr IntType NUM_1024 = 1024; - -// This class is designed to describe multidimensional data: it's type, Layout, Shape and strides -struct KDNN_API_PUBLIC TensorInfo final { - using SizeType = ::KDNN::SizeType; - TensorInfo(const Shape& shape, const Element::Type & t, const Layout& l) noexcept(false) - : dims(shape), type(t), layout(l), stridesInfo() - { - ValidateElementType(type); - ValidateLayout(dims, layout); - SetMinimumRequiredStrides(dims, layout, stridesInfo); - } - - TensorInfo(const Shape& shape, const Element::Type & t, const Layout& l, const Shape& strides) noexcept(false) - : dims(shape), type(t), layout(l), stridesInfo(strides) - { - ValidateElementType(type); - // We use 0 strides for dimensions == 1, - // this done to natively support implicit dimension broadcasting - stridesInfo = Service::FlushStrides(dims, stridesInfo); - ValidateDimsAndStrides(dims, layout, strides); - } - - Shape GetDims() const noexcept - { - return dims; - } - - SizeType GetNumDims() const noexcept - { - return dims.GetNumDims(); - } - - SizeType GetTotalTensorSize() const noexcept - { - return dims.GetTotalDimsSize(); - } - - Shape GetStrides() const noexcept - { - return stridesInfo; - } - - Element::Type GetType() const noexcept - { - return type; - } - - Layout GetLayout() const noexcept - { - return layout; - } - - bool IsDense() const noexcept - { - Shape minStrides = dims; - SetMinimumRequiredStrides(dims, layout, minStrides); - return minStrides == stridesInfo; - } - - bool HasStrideOverFastestDim() const noexcept - { - return (stridesInfo[Service::GetIdxForUserLayout(layout, dims.GetNumDims() - 1)] > 1); - } - - Layout GetStandardABXLayout() const - { - switch (dims.GetNumDims()) { - case DIM_1: { - return Layout::A; - } - case DIM_2: { - return Layout::AB; - } - case DIM_3: { - return Layout::ABC; - } - case DIM_4: { - return Layout::ABCD; - } - case DIM_5: { - return Layout::ABCDE; - } - default: { - // throw Service::LogicError {"Tensor Info: tensor dimensionality is incorrect"}; - } - } - } - - Layout GetStandardAXBLayout() const - { - switch (dims.GetNumDims()) { - case DIM_1: { - return Layout::A; - } - case DIM_2: { - return Layout::AB; - } - case DIM_3: { - return Layout::ACB; - } - case DIM_4: { - return Layout::ACDB; - } - case DIM_5: { - return Layout::ACDEB; - } - default: { - // throw Service::LogicError {"Tensor Info: tensor dimensionality is incorrect"}; - } - } - } - - Layout GetStandardBXALayout() const - { - switch (dims.GetNumDims()) { - case DIM_1: { - return Layout::A; - } - case DIM_2: { - return Layout::BA; - } - case DIM_3: { - return Layout::BCA; - } - case DIM_4: { - return Layout::BCDA; - } - case DIM_5: { - return Layout::BCDEA; - } - default: { - // throw Service::LogicError {"Tensor Info: tensor dimensionality is incorrect"}; - } - } - } - - SizeType GetAxisNumAccordingToLayout(SizeType axisNum) const noexcept(false); - - template - static typename std::conditional::type - ValidateElementType(Element::Type parType) noexcept(areExcetionsDeclined); - - template - static typename std::conditional::type - ValidateLayout(Shape parDims, Layout parLayout) noexcept(areExcetionsDeclined); - - template - static typename std::conditional::type - SetMinimumRequiredStrides(Shape parDims, Layout parLayout, Shape& parStrides) noexcept(areExcetionsDeclined); - - template - static typename std::conditional::type - ValidateDimsAndStrides(Shape parDims, Layout parLayout, Shape parStrides) noexcept(areExcetionsDeclined); - -private: - Shape dims; - Element::Type type; - Layout layout; - Shape stridesInfo; -}; - -inline bool operator==(const TensorInfo &lhs, const TensorInfo &rhs) noexcept -{ - return (lhs.GetDims() == rhs.GetDims()) && - (static_cast(lhs.GetType()) == static_cast(rhs.GetType())) && - (lhs.GetLayout() == rhs.GetLayout()) && (lhs.GetStrides() == rhs.GetStrides()); -} - -inline bool operator!=(const TensorInfo &lhs, const TensorInfo &rhs) noexcept -{ - return !(lhs == rhs); -} - -} // KDNN - -#endif // KDNN_TENSORINFO_HPP diff --git a/third_party/KDNN/kdnn_include/types/kdnn_types.hpp b/third_party/KDNN/kdnn_include/types/kdnn_types.hpp deleted file mode 100644 index 6bafa3aac8c605e1d7697261e673b0a4e877205f..0000000000000000000000000000000000000000 --- a/third_party/KDNN/kdnn_include/types/kdnn_types.hpp +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * Description: Forward declarations of KDNN types for data type abstractions - * Author: KPL - * Create: 2024-06-05 - * Notes: NA - */ - -#ifndef KDNN_TYPES_HPP -#define KDNN_TYPES_HPP - -namespace KDNN { - -struct Type; -struct Shape; -struct TensorInfo; - -} // KDNN - -#endif // KDNN_TYPES_HPP diff --git a/third_party/KDNN/kdnn_lib/libkdnn.so b/third_party/KDNN/kdnn_lib/libkdnn.so deleted file mode 100644 index fe2546938811e8a160590ec464bd88e7ee5c899f..0000000000000000000000000000000000000000 Binary files a/third_party/KDNN/kdnn_lib/libkdnn.so and /dev/null differ diff --git a/third_party/KDNN/kdnn_threadpool.h b/third_party/KDNN/kdnn_threadpool.h index bcbb0b2055ec761715d77bc0583bcb31357e6de4..193a2e0336ef0b2a398bf53a9e98d6073f635095 100644 --- a/third_party/KDNN/kdnn_threadpool.h +++ b/third_party/KDNN/kdnn_threadpool.h @@ -1,19 +1,3 @@ - -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - #ifndef TENSORFLOW_CORE_UTIL_KDNN_THREADPOOL_H_ #define TENSORFLOW_CORE_UTIL_KDNN_THREADPOOL_H_ @@ -50,9 +34,9 @@ class KDNNThreadPool : public ThreadpoolWrapper { virtual bool GetInParallel() const override { return (eigen_interface_->CurrentThreadId() != -1) ? true : false; } - virtual void Parallel(int n, + virtual void Parallel(int n, int cost, const std::function& fn) override { - thread_pool_->ParallelForFixedBlockSizeScheduling(n, 1, fn); + thread_pool_->ParallelFor(n, cost, fn); } ~KDNNThreadPool() {}