diff --git a/benchmark/ascend/bert/src/bert_model.py b/benchmark/ascend/bert/src/bert_model.py index 2923199fc7230614edd7b3435296ba7e81707bbd..9aa09f39c4a0ef3f99145c8c7c2ef9a7747e51d3 100644 --- a/benchmark/ascend/bert/src/bert_model.py +++ b/benchmark/ascend/bert/src/bert_model.py @@ -27,6 +27,7 @@ from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter from mindspore.ops.operations.nn_ops import FlashAttentionScore +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -340,7 +341,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/community/nlp/Roberta_Seq2Seq/src/roberta_model.py b/community/nlp/Roberta_Seq2Seq/src/roberta_model.py index e831ead2ee49e98fe475108b7bf63aaadefc2ef9..ee15032105ad0c10f2a696a7db11a45be8c25101 100644 --- a/community/nlp/Roberta_Seq2Seq/src/roberta_model.py +++ b/community/nlp/Roberta_Seq2Seq/src/roberta_model.py @@ -28,6 +28,7 @@ from mindspore import log as logger import mindspore.common.dtype as mstype from mindspore.common import initializer as init from src.utils import generate_arange_tensor +from utils.ms_tools.ms_tools import dtype_to_nptype ACT2FN = {"gelu": ops.GeLU(), "relu": ops.ReLU(), "swish": ops.HSwish(), "mish": ops.Mish()} @@ -179,7 +180,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/official/nlp/Bert/src/bert_model.py b/official/nlp/Bert/src/bert_model.py index 3d50b026b1c755f49c25df3dd4e85ffcbec21217..de8af35a17163d2e5129ef01e808d80e51339b67 100644 --- a/official/nlp/Bert/src/bert_model.py +++ b/official/nlp/Bert/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -339,7 +340,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/official/nlp/Bert_thor/src/bert_model.py b/official/nlp/Bert_thor/src/bert_model.py index 4390df458dd5f13456786fc2dcd230d94d7694a1..c0cbcb0105970d606a9f30304163c5dec6bc1047 100644 --- a/official/nlp/Bert_thor/src/bert_model.py +++ b/official/nlp/Bert_thor/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -341,7 +342,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/DYR/src/bert_model.py b/research/nlp/DYR/src/bert_model.py index b176000d3dc09b15882d2a018a7b82be98e3487f..a75bbbc84ab6c4ab92c050f6f671686872b15e00 100644 --- a/research/nlp/DYR/src/bert_model.py +++ b/research/nlp/DYR/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -341,7 +342,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/albert/src/albert_model.py b/research/nlp/albert/src/albert_model.py index e267ce4730045fd0822bcdc98b02919eaeafd556..bf6587777ffa180e75e56db92ccd7e4c854bdac7 100644 --- a/research/nlp/albert/src/albert_model.py +++ b/research/nlp/albert/src/albert_model.py @@ -27,6 +27,7 @@ from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter from mindspore.ops.primitive import Primitive from mindspore._extends import cell_attr_register +from utils.ms_tools.ms_tools import dtype_to_nptype class AlbertConfig: @@ -371,7 +372,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = Tensor(np.finfo(np_type).min, dtype=src_type) self.tensor_max_type = Tensor(np.finfo(np_type).max, dtype=src_type) diff --git a/research/nlp/cpm/src/model_cpm.py b/research/nlp/cpm/src/model_cpm.py index be16acfeaf73b02d6d55fe626ce33ab585efdc38..49a2f822c9dbf37ee0deb8e07a205769fb1ded08 100644 --- a/research/nlp/cpm/src/model_cpm.py +++ b/research/nlp/cpm/src/model_cpm.py @@ -19,9 +19,9 @@ from mindspore.train.callback import RunContext from mindspore import context from mindspore.context import ParallelMode from mindspore import Model, connect_network_with_dataset -from mindspore.common.dtype import _pytype_to_dtype from mindspore._c_expression import init_exec_dataset from mindspore.train.train_thor.dataset_helper import DatasetHelper +from utils.ms_tools.ms_tools import pytype_to_dtype def _convert_type(types): @@ -36,7 +36,7 @@ def _convert_type(types): """ ms_types = [] for np_type in types: - ms_type = _pytype_to_dtype(np_type) + ms_type = pytype_to_dtype(np_type) ms_types.append(ms_type) return ms_types diff --git a/research/nlp/dgu/src/bert_model.py b/research/nlp/dgu/src/bert_model.py index 2c6a22946b047afdda36e74a873b620cefd2f0e0..6a10cc607b57a7abf6561903f187c9aa1a283a76 100644 --- a/research/nlp/dgu/src/bert_model.py +++ b/research/nlp/dgu/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -349,7 +350,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/emotect/src/ernie_model.py b/research/nlp/emotect/src/ernie_model.py index 864cfde96ffb65a8e04216a44a654a41d1561907..233715d3d97af4438ac564ec0bf086030dd7fb79 100644 --- a/research/nlp/emotect/src/ernie_model.py +++ b/research/nlp/emotect/src/ernie_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class ErnieConfig: @@ -334,7 +335,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/ernie/src/ernie_model.py b/research/nlp/ernie/src/ernie_model.py index 83c7cab06d8a13b0e05ce29ce85ca24013fe2458..f54aef08a529ddd22caaecad196c1ec33a88a557 100644 --- a/research/nlp/ernie/src/ernie_model.py +++ b/research/nlp/ernie/src/ernie_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class ErnieConfig: @@ -333,7 +334,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/ktnet/src/bert.py b/research/nlp/ktnet/src/bert.py index 2aae158188f69d13b8d85f31988556d845bba0a5..aeeab792f5da151aa88e06e019c63cb6d1f0edae 100644 --- a/research/nlp/ktnet/src/bert.py +++ b/research/nlp/ktnet/src/bert.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -356,7 +357,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/senta/src/bert_model.py b/research/nlp/senta/src/bert_model.py index f3366f83cc39e45e825f43523cc8a8082b1fe57d..cac8a36de9641c94722503547de3d842bc4cc8a9 100644 --- a/research/nlp/senta/src/bert_model.py +++ b/research/nlp/senta/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -349,7 +350,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/soft_masked_bert/src/bert_model.py b/research/nlp/soft_masked_bert/src/bert_model.py index d9e11ea49d7730c6683c54fffae1918a2f108784..51bb479a787959308831e17b3dd4db9c21cc867f 100644 --- a/research/nlp/soft_masked_bert/src/bert_model.py +++ b/research/nlp/soft_masked_bert/src/bert_model.py @@ -27,6 +27,7 @@ from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter import mindspore.ops as ops from mindspore.train.serialization import load_param_into_net +from utils.ms_tools.ms_tools import dtype_to_nptype class BertOnlyMLMHead(nn.Cell): @@ -400,7 +401,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/ternarybert/src/tinybert_model.py b/research/nlp/ternarybert/src/tinybert_model.py index 6f3bb21c0e269103b47c744f828f382027fb196a..4166e9d91ac19f9156b4be751123de9a0d652f0f 100644 --- a/research/nlp/ternarybert/src/tinybert_model.py +++ b/research/nlp/ternarybert/src/tinybert_model.py @@ -29,6 +29,7 @@ from mindspore.common.parameter import Parameter from mindspore import context from mindspore.nn.layer.quant import FakeQuantWithMinMaxObserver as FakeQuantWithMinMax from mindspore.nn.layer import get_activation +from utils.ms_tools.ms_tools import dtype_to_nptype class DenseGeLU(nn.Cell): @@ -564,7 +565,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) min_type = np.finfo(np_type).min max_type = np.finfo(np_type).max self.tensor_min_type = Tensor([min_type], dtype=src_type) diff --git a/research/nlp/tinybert/src/tinybert_model.py b/research/nlp/tinybert/src/tinybert_model.py index 1b4cc2de9821c9c42cfeb594bf06575f8284bdc5..52b44ff4a0108f465af58ba4dcf9df01e3021470 100644 --- a/research/nlp/tinybert/src/tinybert_model.py +++ b/research/nlp/tinybert/src/tinybert_model.py @@ -25,6 +25,7 @@ from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter from mindspore import context +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -298,7 +299,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) min_type = np.finfo(np_type).min max_type = np.finfo(np_type).max self.tensor_min_type = Tensor([min_type], dtype=src_type) diff --git a/utils/ms_tools/ms_tools.py b/utils/ms_tools/ms_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..bc1ca6c96f2c6203440240ec8ec29945a5c36a98 --- /dev/null +++ b/utils/ms_tools/ms_tools.py @@ -0,0 +1,91 @@ +# Copyright 2025 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""mindspore tools.""" +import typing +import numpy as np +import mindspore.common.dtype as mstype +from mindspore._c_expression.np_dtypes import np_version_valid + +if np_version_valid(False): + from mindspore._c_expression.np_dtypes import bfloat16 as np_bfloat16 + +_dtype_nptype_dict = { + mstype.bool_: np.bool_, + mstype.int8: np.int8, + mstype.int16: np.int16, + mstype.int32: np.int32, + mstype.int64: np.int64, + mstype.uint8: np.uint8, + mstype.uint16: np.uint16, + mstype.uint32: np.uint32, + mstype.uint64: np.uint64, + mstype.float16: np.float16, + mstype.float32: np.float32, + mstype.float64: np.float64, + mstype.complex64: np.complex64, + mstype.complex128: np.complex128, +} + +_simple_types = { + list: mstype.list_, + tuple: mstype.tuple_, + type(None): mstype.type_none, + bool: mstype.bool_, + int: mstype.int64, + float: mstype.float64, + complex: mstype.complex128, + str: mstype.string, + np.bool_: mstype.bool_, + np.str_: mstype.string, + np.int8: mstype.int8, + np.int16: mstype.int16, + np.int32: mstype.int32, + np.int64: mstype.int64, + np.uint8: mstype.uint8, + np.uint16: mstype.uint16, + np.uint32: mstype.uint32, + np.uint64: mstype.uint64, + np.float16: mstype.float16, + np.float32: mstype.float32, + np.float64: mstype.float64, +} + +def dtype_to_nptype(type_): + """ + Convert MindSpore dtype to numpy data type. + """ + if type_ == mstype.bfloat16: + if not np_version_valid(True): + raise TypeError("The Numpy bfloat16 data type is not supported now, please ensure that the current " + "Numpy version is not less than the version when the mindspore is compiled, " + "and the major versions are same.") + return np_bfloat16 + return _dtype_nptype_dict[type_] + + +def pytype_to_dtype(obj): + """ + Convert python type to MindSpore type. + """ + if isinstance(obj, np.dtype): + obj = obj.type + if isinstance(obj, typing.Type): + return obj + if not isinstance(obj, type): + raise TypeError("For 'pytype_to_dtype', the argument 'obj' must be a python type object," + "such as int, float, str, etc. But got type {}.".format(type(obj))) + if obj in _simple_types: + return _simple_types[obj] + raise NotImplementedError(f"The python type {obj} cannot be converted to MindSpore type.")