From 8637cf78d8c436b1e8b47f0b97efb83aaa2910cc Mon Sep 17 00:00:00 2001 From: guozhibin Date: Thu, 17 Jul 2025 10:14:09 +0800 Subject: [PATCH] adapte new ms dtype --- benchmark/ascend/bert/src/bert_model.py | 3 +- .../nlp/Roberta_Seq2Seq/src/roberta_model.py | 3 +- official/nlp/Bert/src/bert_model.py | 3 +- official/nlp/Bert_thor/src/bert_model.py | 3 +- research/nlp/DYR/src/bert_model.py | 3 +- research/nlp/albert/src/albert_model.py | 3 +- research/nlp/cpm/src/model_cpm.py | 4 +- research/nlp/dgu/src/bert_model.py | 3 +- research/nlp/emotect/src/ernie_model.py | 3 +- research/nlp/ernie/src/ernie_model.py | 3 +- research/nlp/ktnet/src/bert.py | 3 +- research/nlp/senta/src/bert_model.py | 3 +- .../nlp/soft_masked_bert/src/bert_model.py | 3 +- .../nlp/ternarybert/src/tinybert_model.py | 3 +- research/nlp/tinybert/src/tinybert_model.py | 3 +- utils/ms_tools/ms_tools.py | 91 +++++++++++++++++++ 16 files changed, 121 insertions(+), 16 deletions(-) create mode 100644 utils/ms_tools/ms_tools.py diff --git a/benchmark/ascend/bert/src/bert_model.py b/benchmark/ascend/bert/src/bert_model.py index 2923199fc..9aa09f39c 100644 --- a/benchmark/ascend/bert/src/bert_model.py +++ b/benchmark/ascend/bert/src/bert_model.py @@ -27,6 +27,7 @@ from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter from mindspore.ops.operations.nn_ops import FlashAttentionScore +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -340,7 +341,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/community/nlp/Roberta_Seq2Seq/src/roberta_model.py b/community/nlp/Roberta_Seq2Seq/src/roberta_model.py index e831ead2e..ee1503210 100644 --- a/community/nlp/Roberta_Seq2Seq/src/roberta_model.py +++ b/community/nlp/Roberta_Seq2Seq/src/roberta_model.py @@ -28,6 +28,7 @@ from mindspore import log as logger import mindspore.common.dtype as mstype from mindspore.common import initializer as init from src.utils import generate_arange_tensor +from utils.ms_tools.ms_tools import dtype_to_nptype ACT2FN = {"gelu": ops.GeLU(), "relu": ops.ReLU(), "swish": ops.HSwish(), "mish": ops.Mish()} @@ -179,7 +180,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/official/nlp/Bert/src/bert_model.py b/official/nlp/Bert/src/bert_model.py index 3d50b026b..de8af35a1 100644 --- a/official/nlp/Bert/src/bert_model.py +++ b/official/nlp/Bert/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -339,7 +340,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/official/nlp/Bert_thor/src/bert_model.py b/official/nlp/Bert_thor/src/bert_model.py index 4390df458..c0cbcb010 100644 --- a/official/nlp/Bert_thor/src/bert_model.py +++ b/official/nlp/Bert_thor/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -341,7 +342,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/DYR/src/bert_model.py b/research/nlp/DYR/src/bert_model.py index b176000d3..a75bbbc84 100644 --- a/research/nlp/DYR/src/bert_model.py +++ b/research/nlp/DYR/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -341,7 +342,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/albert/src/albert_model.py b/research/nlp/albert/src/albert_model.py index e267ce473..bf6587777 100644 --- a/research/nlp/albert/src/albert_model.py +++ b/research/nlp/albert/src/albert_model.py @@ -27,6 +27,7 @@ from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter from mindspore.ops.primitive import Primitive from mindspore._extends import cell_attr_register +from utils.ms_tools.ms_tools import dtype_to_nptype class AlbertConfig: @@ -371,7 +372,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = Tensor(np.finfo(np_type).min, dtype=src_type) self.tensor_max_type = Tensor(np.finfo(np_type).max, dtype=src_type) diff --git a/research/nlp/cpm/src/model_cpm.py b/research/nlp/cpm/src/model_cpm.py index be16acfea..49a2f822c 100644 --- a/research/nlp/cpm/src/model_cpm.py +++ b/research/nlp/cpm/src/model_cpm.py @@ -19,9 +19,9 @@ from mindspore.train.callback import RunContext from mindspore import context from mindspore.context import ParallelMode from mindspore import Model, connect_network_with_dataset -from mindspore.common.dtype import _pytype_to_dtype from mindspore._c_expression import init_exec_dataset from mindspore.train.train_thor.dataset_helper import DatasetHelper +from utils.ms_tools.ms_tools import pytype_to_dtype def _convert_type(types): @@ -36,7 +36,7 @@ def _convert_type(types): """ ms_types = [] for np_type in types: - ms_type = _pytype_to_dtype(np_type) + ms_type = pytype_to_dtype(np_type) ms_types.append(ms_type) return ms_types diff --git a/research/nlp/dgu/src/bert_model.py b/research/nlp/dgu/src/bert_model.py index 2c6a22946..6a10cc607 100644 --- a/research/nlp/dgu/src/bert_model.py +++ b/research/nlp/dgu/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -349,7 +350,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/emotect/src/ernie_model.py b/research/nlp/emotect/src/ernie_model.py index 864cfde96..233715d3d 100644 --- a/research/nlp/emotect/src/ernie_model.py +++ b/research/nlp/emotect/src/ernie_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class ErnieConfig: @@ -334,7 +335,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/ernie/src/ernie_model.py b/research/nlp/ernie/src/ernie_model.py index 83c7cab06..f54aef08a 100644 --- a/research/nlp/ernie/src/ernie_model.py +++ b/research/nlp/ernie/src/ernie_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class ErnieConfig: @@ -333,7 +334,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/ktnet/src/bert.py b/research/nlp/ktnet/src/bert.py index 2aae15818..aeeab792f 100644 --- a/research/nlp/ktnet/src/bert.py +++ b/research/nlp/ktnet/src/bert.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -356,7 +357,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/senta/src/bert_model.py b/research/nlp/senta/src/bert_model.py index f3366f83c..cac8a36de 100644 --- a/research/nlp/senta/src/bert_model.py +++ b/research/nlp/senta/src/bert_model.py @@ -26,6 +26,7 @@ from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -349,7 +350,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/soft_masked_bert/src/bert_model.py b/research/nlp/soft_masked_bert/src/bert_model.py index d9e11ea49..51bb479a7 100644 --- a/research/nlp/soft_masked_bert/src/bert_model.py +++ b/research/nlp/soft_masked_bert/src/bert_model.py @@ -27,6 +27,7 @@ from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter import mindspore.ops as ops from mindspore.train.serialization import load_param_into_net +from utils.ms_tools.ms_tools import dtype_to_nptype class BertOnlyMLMHead(nn.Cell): @@ -400,7 +401,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) diff --git a/research/nlp/ternarybert/src/tinybert_model.py b/research/nlp/ternarybert/src/tinybert_model.py index 6f3bb21c0..4166e9d91 100644 --- a/research/nlp/ternarybert/src/tinybert_model.py +++ b/research/nlp/ternarybert/src/tinybert_model.py @@ -29,6 +29,7 @@ from mindspore.common.parameter import Parameter from mindspore import context from mindspore.nn.layer.quant import FakeQuantWithMinMaxObserver as FakeQuantWithMinMax from mindspore.nn.layer import get_activation +from utils.ms_tools.ms_tools import dtype_to_nptype class DenseGeLU(nn.Cell): @@ -564,7 +565,7 @@ class SaturateCast(nn.Cell): def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) min_type = np.finfo(np_type).min max_type = np.finfo(np_type).max self.tensor_min_type = Tensor([min_type], dtype=src_type) diff --git a/research/nlp/tinybert/src/tinybert_model.py b/research/nlp/tinybert/src/tinybert_model.py index 1b4cc2de9..52b44ff4a 100644 --- a/research/nlp/tinybert/src/tinybert_model.py +++ b/research/nlp/tinybert/src/tinybert_model.py @@ -25,6 +25,7 @@ from mindspore.ops import composite as C from mindspore.common.tensor import Tensor from mindspore.common.parameter import Parameter from mindspore import context +from utils.ms_tools.ms_tools import dtype_to_nptype class BertConfig: @@ -298,7 +299,7 @@ class SaturateCast(nn.Cell): """ def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() - np_type = mstype._dtype_to_nptype(dst_type) # pylint: disable=protected-access + np_type = dtype_to_nptype(dst_type) min_type = np.finfo(np_type).min max_type = np.finfo(np_type).max self.tensor_min_type = Tensor([min_type], dtype=src_type) diff --git a/utils/ms_tools/ms_tools.py b/utils/ms_tools/ms_tools.py new file mode 100644 index 000000000..bc1ca6c96 --- /dev/null +++ b/utils/ms_tools/ms_tools.py @@ -0,0 +1,91 @@ +# Copyright 2025 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""mindspore tools.""" +import typing +import numpy as np +import mindspore.common.dtype as mstype +from mindspore._c_expression.np_dtypes import np_version_valid + +if np_version_valid(False): + from mindspore._c_expression.np_dtypes import bfloat16 as np_bfloat16 + +_dtype_nptype_dict = { + mstype.bool_: np.bool_, + mstype.int8: np.int8, + mstype.int16: np.int16, + mstype.int32: np.int32, + mstype.int64: np.int64, + mstype.uint8: np.uint8, + mstype.uint16: np.uint16, + mstype.uint32: np.uint32, + mstype.uint64: np.uint64, + mstype.float16: np.float16, + mstype.float32: np.float32, + mstype.float64: np.float64, + mstype.complex64: np.complex64, + mstype.complex128: np.complex128, +} + +_simple_types = { + list: mstype.list_, + tuple: mstype.tuple_, + type(None): mstype.type_none, + bool: mstype.bool_, + int: mstype.int64, + float: mstype.float64, + complex: mstype.complex128, + str: mstype.string, + np.bool_: mstype.bool_, + np.str_: mstype.string, + np.int8: mstype.int8, + np.int16: mstype.int16, + np.int32: mstype.int32, + np.int64: mstype.int64, + np.uint8: mstype.uint8, + np.uint16: mstype.uint16, + np.uint32: mstype.uint32, + np.uint64: mstype.uint64, + np.float16: mstype.float16, + np.float32: mstype.float32, + np.float64: mstype.float64, +} + +def dtype_to_nptype(type_): + """ + Convert MindSpore dtype to numpy data type. + """ + if type_ == mstype.bfloat16: + if not np_version_valid(True): + raise TypeError("The Numpy bfloat16 data type is not supported now, please ensure that the current " + "Numpy version is not less than the version when the mindspore is compiled, " + "and the major versions are same.") + return np_bfloat16 + return _dtype_nptype_dict[type_] + + +def pytype_to_dtype(obj): + """ + Convert python type to MindSpore type. + """ + if isinstance(obj, np.dtype): + obj = obj.type + if isinstance(obj, typing.Type): + return obj + if not isinstance(obj, type): + raise TypeError("For 'pytype_to_dtype', the argument 'obj' must be a python type object," + "such as int, float, str, etc. But got type {}.".format(type(obj))) + if obj in _simple_types: + return _simple_types[obj] + raise NotImplementedError(f"The python type {obj} cannot be converted to MindSpore type.") -- Gitee