From 3ed478f8c15009eff60201224875b288c7e7037b Mon Sep 17 00:00:00 2001 From: h00613304 Date: Fri, 15 Mar 2024 10:53:30 +0800 Subject: [PATCH] CCDebugger initial version --- .../CCDebugger/.cloudbuild/build.yml | 28 + .../CCDebugger/.codecheck/check.yml | 11 + debug/accuracy_tools/CCDebugger/README.md | 87 + debug/accuracy_tools/CCDebugger/build.sh | 13 + .../CCDebugger/cc_debugger/__init__.py | 6 + .../cc_debugger/auto_plugging/__init__.py | 3 + .../cc_debugger/auto_plugging/instrument.py | 136 ++ .../cc_debugger/auto_plugging/main.py | 16 + .../CCDebugger/cc_debugger/common/__init__.py | 3 + .../CCDebugger/cc_debugger/common/compare.py | 251 +++ .../cc_debugger/common/custom_iterable.py | 73 + .../CCDebugger/cc_debugger/common/utils.py | 373 ++++ .../CCDebugger/cc_debugger/core/__init__.py | 3 + .../CCDebugger/cc_debugger/core/config.py | 55 + .../CCDebugger/cc_debugger/core/const.py | 85 + .../CCDebugger/cc_debugger/core/enums.py | 97 + .../CCDebugger/cc_debugger/core/exceptions.py | 16 + .../cc_debugger/core/settings/__init__.py | 3 + .../cc_debugger/core/settings/base.env | 24 + .../cc_debugger/core/settings/base.py | 48 + .../cc_debugger/core/settings/gpu.env | 2 + .../cc_debugger/core/settings/npu.env | 8 + .../CCDebugger/cc_debugger/data/torch_api.txt | 25 + .../cc_debugger/data/torch_api_all.txt | 1647 +++++++++++++++++ .../cc_debugger/data/torch_api_blacklists.txt | 7 + .../data/torch_distributed_api.txt | 12 + .../cc_debugger/decorator/__init__.py | 3 + .../cc_debugger/decorator/common.py | 90 + .../cc_debugger/decorator/debugger.py | 112 ++ .../cc_debugger/decorator/dec_backward.py | 263 +++ .../cc_debugger/decorator/dec_forward.py | 150 ++ .../CCDebugger/cc_debugger/layers/__init__.py | 3 + .../cc_debugger/layers/fuzz_layer.py | 252 +++ .../CCDebugger/cc_debugger/main.py | 134 ++ .../CCDebugger/cc_debugger/models/__init__.py | 3 + .../CCDebugger/cc_debugger/models/models.py | 89 + .../cc_debugger/result_handlers/__init__.py | 3 + .../result_handlers/abs_handler.py | 112 ++ .../result_handlers/backward_handler.py | 130 ++ .../result_handlers/check_handler.py | 78 + .../result_handlers/fix_handler.py | 37 + .../result_handlers/handler_factory.py | 65 + .../result_handlers/preheat_handler.py | 197 ++ .../result_handlers/relative_handler.py | 79 + .../cc_debugger/run_api/__init__.py | 0 .../cc_debugger/run_api/run_base.py | 95 + .../cc_debugger/run_api/run_case.py | 88 + .../cc_debugger/statistics/__init__.py | 3 + .../statistics/api_preheat_counter.py | 28 + .../cc_debugger/statistics/api_unused.py | 47 + .../cc_debugger/statistics/api_used.py | 170 ++ .../accuracy_tools/CCDebugger/pyproject.toml | 28 + .../CCDebugger/tests/test_backward.py | 91 + .../CCDebugger/tests/test_distributed_apis.py | 74 + .../CCDebugger/tests/test_forward.py | 91 + .../CCDebugger/tests/test_input_bound.py | 52 + .../CCDebugger/tests/test_perturbation.py | 51 + .../CCDebugger/tests/test_preheat.py | 43 + .../CCDebugger/tests/test_result_handler.py | 44 + 59 files changed, 5737 insertions(+) create mode 100644 debug/accuracy_tools/CCDebugger/.cloudbuild/build.yml create mode 100644 debug/accuracy_tools/CCDebugger/.codecheck/check.yml create mode 100644 debug/accuracy_tools/CCDebugger/README.md create mode 100644 debug/accuracy_tools/CCDebugger/build.sh create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/instrument.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/main.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/common/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/common/compare.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/common/custom_iterable.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/common/utils.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/config.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/const.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/enums.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/exceptions.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.env create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/gpu.env create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/npu.env create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api.txt create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_all.txt create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_blacklists.txt create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_distributed_api.txt create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/decorator/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/decorator/common.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/decorator/debugger.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_backward.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_forward.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/layers/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/layers/fuzz_layer.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/main.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/models/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/models/models.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/abs_handler.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/backward_handler.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/check_handler.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/fix_handler.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/handler_factory.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/preheat_handler.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/relative_handler.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/run_api/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_base.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_case.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/statistics/__init__.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_preheat_counter.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_unused.py create mode 100644 debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_used.py create mode 100644 debug/accuracy_tools/CCDebugger/pyproject.toml create mode 100644 debug/accuracy_tools/CCDebugger/tests/test_backward.py create mode 100644 debug/accuracy_tools/CCDebugger/tests/test_distributed_apis.py create mode 100644 debug/accuracy_tools/CCDebugger/tests/test_forward.py create mode 100644 debug/accuracy_tools/CCDebugger/tests/test_input_bound.py create mode 100644 debug/accuracy_tools/CCDebugger/tests/test_perturbation.py create mode 100644 debug/accuracy_tools/CCDebugger/tests/test_preheat.py create mode 100644 debug/accuracy_tools/CCDebugger/tests/test_result_handler.py diff --git a/debug/accuracy_tools/CCDebugger/.cloudbuild/build.yml b/debug/accuracy_tools/CCDebugger/.cloudbuild/build.yml new file mode 100644 index 000000000..58de102ef --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/.cloudbuild/build.yml @@ -0,0 +1,28 @@ +--- +version: 2.0 + +#构建环境 +env: + resource: + type: docker + image: kweecr05.his.huawei.com:80/ecr-build-green202/nebula_build_green:docker + mode: toolbox + +#构建参数定义, 构建脚本可从环境变量中读取使用这些参数 +params: + - name: product + value: cloudbuild2.0 + +#构建步骤 +steps: + PRE_BUILD: # 构建准备步骤 + - checkout + BUILD: # 构建执行步骤 + - build_execute: + command: sh build.sh + accelerate: false + check: true + POST_BUILD: #构建后步骤 + - upload_cloud_artifact: + file_path: dist/* + - version_set \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/.codecheck/check.yml b/debug/accuracy_tools/CCDebugger/.codecheck/check.yml new file mode 100644 index 000000000..b597adb20 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/.codecheck/check.yml @@ -0,0 +1,11 @@ +--- +version: 2.0 +steps: + pre_codecheck: + - checkout +params: + - name: exclude # 排除检查文件目录 + value: (tests|__init__) +tool_params: + secsolar: + source_dir: ./ \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/README.md b/debug/accuracy_tools/CCDebugger/README.md new file mode 100644 index 000000000..54cce04c2 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/README.md @@ -0,0 +1,87 @@ +# CCDebugger + +--- +该项目用于大模型精度调试中的无标杆问题定位。可以对怀疑的torch算子、自定义函数进行精度测试。 + +所测试的对象的输入需要是tensor或包含tensor的可迭代对象(例如tensor组成的元祖)。 + +注： 当前运行过程中error提示均为调试信息 + +# 功能详解 + +## torch算子添加扰动 + +#### 用法 + +1. CCD工具收集功能准备工作： + ```shell + python cc_debugger/auto_plugging/main.py ccd setup + ``` +2. 修改配置文件（cc_debugger/core/settings/base.env npu.env gpu.env） +3. 启动工具 + - 设置环境变量启动。 + ```shell + # 通过配置环境变量`CCD_FLAG`控制是否进行CCD，CCD_FLAG不配置默认为不进行fuzz扰动, + export CCD_FLAG=true + ``` +4. 正常训练、测试原有模型即可。 + 最终CCD的日志，有误差的算子输入输出都会存放到root_dir路径下 +5. CCD工具收集功能清理工作： + ```shell + python cc_debugger/auto_plugging/main.py ccd teardown + ``` +6. 测试是否连接gpu服务（ccd_mode中有gpu则开启） +```shell +export no_proxy=7.216.52.7 +curl "http://7.216.52.7:8081/hello" +``` + + +#### 配置信息介绍 +- base.env + - ccd_flag: 是否开启ccd + - root_dir: 日志和输出结果存放处 + - log_level: 设置日志的级别（默认ERROR，共三种INFO、ERROR、DEBUG） + - ccd_mode: ccd对比机器，目前支持[npu, cpu, gpu]，可多选,设置不同对比方式的step间隔 + - mode 包含forward和backward,默认forward，分别表示前向输入扰动和反向扰动 + - record_flag: 设置是否开启记录模型所有api功能（True表示开启，False表示关闭，默认关闭） + - handler_type: 指定装饰器功能（对添加扰动后的输入以及原输入进行的操作）check: 记录异常点, fix: 执行修复, relative: 绘制相对误差直方图 + - check 会把异常点时对应的输入、输出、算子名存储到指定位置 + - fix 仅在improve_precision下可用。在该模式下，当高精度计算结果与低精度计算结果存在较大差异时，会使用高精度结果作为算子输出 + - relative 获取相对误差直方图 + - range_fuzz：设置是否在给定step范围内进行扰动（True表示开启， False关闭） + - step_start：设置扰动开始step（含当前step） + - step_end：设置扰动结束step（含当前step） + - recursive_fuzz：是否递归扰动 + - single_fuzz 是否开启一个step只扰动一个算子，默认不开启（当显存不够时，建议开启） +- npu.env + - pert_type: + - add_noise 对原输入添加噪音(对输入加极小值) + - improve_precision 提高精度(bf16 -> f32, f16 -> f32, f32 -> f64) + - no_change 不对输入做任何变化，用于定位不确定算子 + - distributed 对通讯算子加扰动（输入首尾交换） + - is_preheat 是否开启预热开关（默认开启，开启后会根据npu和cpu的对比结果更新误差阈值） + - max_sample： 设置每个预热step的最大采样数（默认为20） + - preheat_step：设置预热step的个数（默认15）(预热阶段更新每个算子对应的阈值) +- gpu.env + - server_url: 设置GPU服务地址 + +## 对自定义函数添加扰动 +1. 安装项目依赖并将cc_debugger目录复制到待测函数同级目录下 +2. 配置文件single_fuzz设置为False +3. 该函数所在文件文件头部添加from cc_debugger.decorator.debugger import decorate_forward_function，并在函数头部添加@decorate_forward_function + 或者from cc_debugger.decorator.debugger import decorate_backward_function，并在函数头部添加@decorate_backward_function + +## 结果解读： +check模式下，该工具会把不符合精度标准（float16，相对误差大于千分之一；float32，相对误差大于万分之一）的算子，及当时触发不满足精度条件的输入、输出全部存储到pt文件夹下，可以通过pickle.load加载。“索引的数据”越偏离1，说明相对误差越大。 +对比结果统计汇总在对应文件夹下的`unequal_result_summary.csv`文件中 + +文件夹命名规则： +`[{pid}] {func_id} {ratio}`(使用空格分隔) +文件夹内数据文件命名规则： +inputs：输入 +fuzz_outputs：fuzz输出（backward模式下为fuzz_grad） +origin_outputs：原始输出（backward模式下为origin_grad） + +## 注记 +- 该项目的算法解析可以参看：http://3ms.huawei.com/km/blogs/details/14756445?l=zh-cn \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/build.sh b/debug/accuracy_tools/CCDebugger/build.sh new file mode 100644 index 000000000..9bc79941e --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +echo "=========== Step: 1.0 - Preparing Build Version ...... " + +datetime=$(date +%m%d%H%M%S) +archive_dir_name=$(grep version pyproject.toml | awk -F'"' '{print $2}') +build_version="${archive_dir_name}_${datetime}" +echo "buildVersion=${build_version}" +echo "buildVersion=${build_version}" >buildInfo.properties +python3 -m pip install --trusted-host mirrors.tools.huawei.com -i https://mirrors.tools.huawei.com/pypi/simple poetry +echo "=========== Step: 2.0 - Build Package For Python...... " +python3 -m poetry build +echo "build success" diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/__init__.py new file mode 100644 index 000000000..2d5efc4a8 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from .main import cc_debugger + +cc_debugger() diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/__init__.py new file mode 100644 index 000000000..c39257b25 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/instrument.py b/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/instrument.py new file mode 100644 index 000000000..64e69445a --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/instrument.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import os +import shutil +import stat +import sys +from enum import Enum +from pathlib import Path +from logging import Logger +from cleo.commands.command import Command +from cleo.helpers import argument +from loguru import logger + + +class ToolName(Enum): + set_up = 'setup' + tear_down = 'teardown' + + +class CCDCommand(Command): + def __init__(self): + + super().__init__() + self.project_path: Path = Path(__file__).parent.parent + self.log_name: str = 'torch_plugging' + self.log_path = self.project_path.joinpath('logs').joinpath(self.log_name + '_{time}.log') + self.logger: Logger = self._set_logger() + self.ccd_plugging_dir: Path = self.project_path + + name = "ccd" + description = "Collection operator api info" + arguments = [ + argument('name', + description='Setup or teardown the ccd tool. support value: setup, teardown. Default value is setup.', + optional=True, + default='setup') + ] + + @staticmethod + def get_del_line(fp, tear_str): + del_lines = [] + file_line_num = 0 + for i, line in enumerate(fp): + if line.strip() == tear_str: + del_lines.append(i) + file_line_num = i + if not del_lines: + raise ValueError("Not Found") + del_line = del_lines[-1] + if not tear_str and file_line_num - del_line > 5: + raise ValueError("Not Found") + return del_line + + @staticmethod + def tear_down(file_path, tear_str): + with open(file_path, 'r') as old_file: + try: + del_line = CCDCommand.get_del_line(old_file, tear_str) + except ValueError: + return + old_file.seek(0) + with open(file_path, 'r+') as new_file: + current_line = 0 + while current_line < del_line: + old_file.readline() + current_line += 1 + + seek_point = old_file.tell() + new_file.seek(seek_point, 0) + old_file.readline() + next_line = old_file.readline() + + while next_line: + new_file.write(next_line) + next_line = old_file.readline() + new_file.truncate() + + def set_up(self, file_path, setup_str): + # 将ccd代码放入torch源码中 + torch_dir = Path(file_path).parent + self._move_ccd_plugging_dir(torch_dir) + flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND + mode = stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRWXG | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + with open(file_path, 'r') as f: + for line in f: + if setup_str == line.strip(): + return + with os.fdopen(os.open(file_path, flags, mode), 'a+') as f: + f.write("\n") + f.write(setup_str) + + def handle(self): + try: + tool_name = ToolName(self.argument("name")) + except ValueError as e: + raise ValueError( + f"Not Found {self.argument('name')}. expected value in {[t.value for t in ToolName]}") from e + _torch_path = None + for p in sys.path: + if os.path.isdir(p) and 'torch' in os.listdir(p): + _torch_path = os.path.join(p, 'torch') + if not _torch_path: + raise FileNotFoundError("Not Found torch") + _torch_init = os.path.join(_torch_path, '__init__.py') + if not os.path.exists(_torch_init): + raise FileNotFoundError("Not Found torch/__init__.py") + _instrument_str = "import cc_debugger" + if tool_name == ToolName.set_up: + self.set_up(_torch_init, _instrument_str) + if tool_name == ToolName.tear_down: + CCDCommand.tear_down(_torch_init, _instrument_str) + CCDCommand.tear_down(_torch_init, '') + + def _set_logger(self) -> Logger: + """ + 设置日志记录器 + + :return: + """ + logger.add(self.log_path.as_posix(), filter=lambda record: record['extra']['name'] == self.log_name) + return logger.bind(name=self.log_name) + + def _move_ccd_plugging_dir(self, torch_root_dir): + """ + 将ccd_debugger文件夹复制到Pytorch包根目录下面 + :param torch_root_dir: torch的根路径 + + :return: + """ + torch_ccd_plugging_dir = torch_root_dir.parent.joinpath("cc_debugger") + # 3.7版本的python的shutil.copytree不支持参数dirs_exist_ok 需要先将文件夹删除再复制过去 + if os.path.exists(torch_ccd_plugging_dir): + shutil.rmtree(torch_ccd_plugging_dir) + shutil.copytree(self.ccd_plugging_dir, torch_ccd_plugging_dir) + self.logger.info('The directory [cc_debugger] has been copied.') diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/main.py b/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/main.py new file mode 100644 index 000000000..18a294fb3 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/auto_plugging/main.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from cleo.application import Application + +from cc_debugger.auto_plugging.instrument import CCDCommand + + +def main(): + application = Application() + application.add(CCDCommand()) + application.run() + + +if __name__ == "__main__": + main() diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/common/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/common/__init__.py new file mode 100644 index 000000000..601aa4d99 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/common/__init__.py @@ -0,0 +1,3 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/common/compare.py b/debug/accuracy_tools/CCDebugger/cc_debugger/common/compare.py new file mode 100644 index 000000000..bd43ccb13 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/common/compare.py @@ -0,0 +1,251 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Tuple + +import numpy as np +import pandas as pd +import torch +from loguru import logger +from numpy import ndarray +from torch import Tensor +from torch._C import _VariableFunctions as torchCFunctions + + +class CommonField: + INDEX: str = "index" + MIN_: str = "min" + MAX_: str = "max" + MEAN_: str = "mean" + COUNT: str = "count" + INPUT: str = "input" + TARGET: str = "target" + + +class PrecisionConfig: + THD_TENSOR = { + torch.float64: (0.0001, 0.0001), + torch.float32: (0.0001, 0.0001), + torch.float16: (0.001, 0.001), + torch.bfloat16: (0.001, 0.001) + } + THD_NUMPY = { + "float32": (0.0001, 0.0001), + "float16": (0.001, 0.001) + } + MAX_DIFF_HD = 0.1 + DISPLAY_NUM = 10 + MAX_ERROR_IDX = 10000000 + + +class TensorCompare: + def __init__(self, expect_data: Tensor, real_data: Tensor, diff_thd=0.001, pct_thd=0.001): + self.expect_data = expect_data + self.real_data = real_data + + self.r_diff = None + self.df = None + + self.diff_thd = diff_thd + self.pct_thd = pct_thd + + @staticmethod + def replace_overflow(tensor: Tensor): + # Replace the overflow data with the minimum or maximum value + if torch.is_floating_point(tensor): + tensor.clamp_(min=torch.finfo(tensor.dtype).min, max=torch.finfo(tensor.dtype).max) + else: + tensor.clamp_(min=torch.iinfo(tensor.dtype).min, max=torch.iinfo(tensor.dtype).max) + + @staticmethod + def compare_equal(expect_tensor: Tensor, real_tensor: Tensor) -> bool: + """ + for not float tensors, compare directly + :param expect_tensor: + :param real_tensor: + :return: + """ + if torch.is_floating_point(expect_tensor): + logger.warning("It's not advised to directly use eq to compare {}", expect_tensor.dtype) + + return torchCFunctions.equal(expect_tensor, real_tensor) + + def data_pre_check(self) -> Tuple[bool, bool]: + """ + pre compare two data, such as size, overflow and others + :return: need next check? pass the check? + """ + if self.real_data.shape != self.expect_data.shape: + logger.error("The shape is not equal: {} {}", self.real_data.shape, self.expect_data.shape) + return False, False + + if self.real_data.numel() == 0 and self.expect_data.numel() == 0: + logger.warning('The tensor is empty, so pass') + return False, True + + overflows_count = torchCFunctions.sum(torchCFunctions.isinf(self.expect_data)) + \ + torchCFunctions.sum(torchCFunctions.isnan(self.expect_data)) + if overflows_count > 0: + logger.warning('Overflow, num:{}/{}', overflows_count, self.real_data.numel()) + if overflows_count == self.real_data.numel(): + return False, self._compare_overflow() + logger.warning('Replace overflow point by max or min') + self.replace_overflow(self.expect_data) + self.replace_overflow(self.real_data) + return True, False + + def compare_tensor(self) -> bool: + self.expect_data = self.expect_data.flatten() + self.real_data = self.real_data.flatten() + + self.r_diff = self._cal_relative_diff_tensor(self.expect_data, self.real_data) + # A single-point function is abnormal + diff_index = torchCFunctions.where(self.r_diff > PrecisionConfig.MAX_DIFF_HD, 1, 0) + if torchCFunctions.sum(diff_index) > 0: + logger.error('Single point r_diff > {}, total num: {}/{}', PrecisionConfig.MAX_DIFF_HD, + torchCFunctions.sum(diff_index), self.real_data.numel()) + self._display_diff_df(diff_index) + return False + # Double thousand or double ten thousand + diff_index = torchCFunctions.where(self.r_diff < self.pct_thd, 0, 1) + second_threshold = torchCFunctions.sum(diff_index) / self.real_data.numel() + if second_threshold > self.pct_thd: + logger.error("Total r_diff percentage: [{}] > [{}]", second_threshold, self.pct_thd) + self._display_diff_df(diff_index) + return False + return True + + def _compare_overflow(self) -> bool: + is_nan_expect = torchCFunctions.isnan(self.expect_data) + is_nan_real = torchCFunctions.isnan(self.real_data) + is_nan_equal = torchCFunctions.all(is_nan_real == is_nan_expect).item() + + is_inf_expect = torchCFunctions.isinf(self.expect_data) + is_inf_real = torchCFunctions.isinf(self.real_data) + is_inf_equal = torchCFunctions.all(is_inf_real == is_inf_expect).item() + + return is_inf_equal and is_nan_equal + + def _display_diff_df(self, diff_index: Tensor): + """ + print the top relative difference lines + :param diff_index: the result of torch where + :return: + """ + + self.df = pd.DataFrame(columns=[CommonField.INDEX, 'ExpectOut', 'RealOut', 'FpDiff', 'RateDiff']) + fp = torchCFunctions.abs(torchCFunctions.subtract(self.real_data, self.expect_data)) + indices = torchCFunctions.where(diff_index > 0)[0] + display_num = len(indices) if len(indices) < PrecisionConfig.DISPLAY_NUM else PrecisionConfig.DISPLAY_NUM + for i in range(display_num): + index = indices[i] + self.df.loc[i] = [ + index.item(), self.expect_data[index].item(), self.real_data[index].item(), fp[index].item(), + self.r_diff[index].item() + ] + self.df[CommonField.INDEX] = self.df[CommonField.INDEX].astype(int) + + logger.error("Failed:\n{}", self.df) + + def _cal_relative_diff_tensor(self, expect_data, real_data): + """ + calculate the relative difference of two flattened tensor. + If the difference is less than the threshold, + the absolute difference is used. + Otherwise, the relative difference is used. + + :param expect_data: expect data to float + :param real_data: real data to float + :return: the relative difference + """ + a = torchCFunctions.abs(real_data - expect_data) + b1 = torchCFunctions.maximum(torchCFunctions.abs(real_data), torchCFunctions.abs(expect_data)) + # avoid div zero exception + b = torchCFunctions.add(b1, 10e-10) + # if absolute diff < 0.001, use absolute diff, otherwise use relative diff + result = torchCFunctions.where(a < self.diff_thd, a, a / b) + return result + + +def data_compare_tensor(expect_tensor: Tensor, real_tensor: Tensor) -> bool: + """ + compare the precision of two tensor + :return: + """ + if not torch.is_floating_point(expect_tensor): + return TensorCompare.compare_equal(expect_tensor, real_tensor) + try: + thd = PrecisionConfig.THD_TENSOR[expect_tensor.dtype] + tc = TensorCompare(expect_tensor, real_tensor, *thd) + except KeyError: + logger.warning("{} not in threshold settings, use default thd value", real_tensor.dtype) + tc = TensorCompare(expect_tensor, real_tensor) + next_check_flag, res = tc.data_pre_check() + if next_check_flag: + return tc.compare_tensor() + return res + + +def cal_relative_diff_np(real_data, expect_data, diff_thd): + a = np.abs(np.subtract(real_data, expect_data)) + b1 = np.maximum(np.abs(real_data), (np.abs(expect_data))) + b2 = float((1.0 / (1 << 14)) / diff_thd) + b = np.add(np.maximum(b1, b2), 10e-10) + result = np.where(a < diff_thd, a, a / b) + return result + + +def data_compare_np(npu_output, cpu_output, diff_thd=0.001, pct_thd=0.001, max_diff_hd=0.1) -> bool: + max_error_idx = 10000000 + real_data = npu_output.flatten() + data_comp = cpu_output.flatten() + if real_data.size == 0 and real_data.size == data_comp.size: + logger.info('The npu_output is [],and it is same as bm_output, the result of data_compare is \"Pass\"') + return True + start = 0 + end = real_data.size - 1 + if end < start: + end = start + max_error = 0 + if real_data.size != data_comp.size: + logger.error('the size of npu output {} and run_api {} is not equal.', real_data.size, data_comp.size) + return False + + overflows_count = data_comp[np.isinf(data_comp)].size + data_comp[np.isnan(data_comp)].size + if overflows_count > 0: + logger.warning('Overflow,size:{},benchmark_output:{}, {}', + overflows_count, data_comp[np.isinf(data_comp)][0:10], data_comp[np.isnan(data_comp)][0:10]) + + split_count = int(end - start + 1) if end != start else 1 + logger.info('split_count:{}; max_diff_hd:{};', float(split_count), max_diff_hd) + + diff_abs = np.abs(np.subtract(real_data.astype(np.float32), data_comp.astype(np.float32))) + + diff_index = np.where(diff_abs > 0) + rdiff = cal_relative_diff_np(real_data[diff_index].astype(np.float32), + data_comp[diff_index].astype(np.float32), + diff_thd) + err_diff = rdiff[rdiff > diff_thd] + fulfill_percent = float(split_count - err_diff.size) / float(split_count) * 100.0 + + pct_thd = (1 - pct_thd) * 100.0 + result = True if (fulfill_percent >= pct_thd) else False + if len(err_diff) > 0: + max_error = max(err_diff[0:max_error_idx]) + if max_error >= max_diff_hd: + result = False + + if len(err_diff) > 0: + logger.error('Max-RelativeError is: {}. Threshold is: {}.', max_error, max_diff_hd) + return result + + +def data_compare_numpy(expect_numpy: ndarray, real_numpy: ndarray) -> bool: + if not np.issubdtype(expect_numpy.dtype, np.floating): + return np.array_equal(expect_numpy, real_numpy) + try: + thd = PrecisionConfig.THD_NUMPY[str(expect_numpy.dtype)] + return data_compare_np(expect_numpy, real_numpy, *thd, PrecisionConfig.MAX_DIFF_HD) + except KeyError: + logger.warning("{} is not in thd settings, use default", expect_numpy.dtype) + return data_compare_np(expect_numpy, real_numpy) diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/common/custom_iterable.py b/debug/accuracy_tools/CCDebugger/cc_debugger/common/custom_iterable.py new file mode 100644 index 000000000..5bc6c546c --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/common/custom_iterable.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +自定义可迭代对象 + +""" +from typing import Iterable, List, Set, Tuple, Dict + +from loguru import logger + +from cc_debugger.core.exceptions import UnsupportedTypeException + + +class CustomIterable: + + def __init__(self, iterable: Iterable, func_id: str): + self.iterable: Iterable = iterable + self.container = self._create() + self.func_id: str = func_id + + def append(self, v: object): + """ + 添加元素 + + :param v: + :return: + """ + if self.container is None: + logger.error('[{}] CustomIterable container is null.', self.func_id) + return + if isinstance(self.container, List): + self.container.append(v) + elif isinstance(self.container, Set): + self.container.add(v) + else: + logger.error('[{}] Unsupported types. type: {}.', self.func_id, self.container) + return + + def add_pair(self, key, value): + if self.container is None: + logger.error('[{}] CustomIterable container is null.', self.func_id) + return + if isinstance(self.container, Dict): + self.container[key] = value + else: + logger.error('[{}] Unsupported types. type: {}.', self.func_id, self.container) + return + + def get_container(self): + """ + 获取容器 + + :return: + """ + if isinstance(self.iterable, Tuple): + return tuple(self.container) + else: + return self.container + + def _create(self): + """ + 构建对应的容器 + + :return: + """ + if isinstance(self.iterable, (List, Tuple)): + return list() + elif isinstance(self.iterable, Dict): + return {} + else: + logger.error('[{}] Unsupported types. type: {}.', self.func_id, self.iterable) + raise UnsupportedTypeException() diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/common/utils.py b/debug/accuracy_tools/CCDebugger/cc_debugger/common/utils.py new file mode 100644 index 000000000..2388e1d7b --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/common/utils.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import math +import os +import stat +import csv +import pickle +import sys +import tempfile +import traceback +from pathlib import Path +from typing import List, Tuple, Any, Dict +import torch +from loguru import logger +from torch import Tensor +from torch._C import _VariableFunctions as torchCFunctions +from cc_debugger.common.compare import data_compare_tensor +from cc_debugger.core.config import base_settings +from cc_debugger.core.const import ThdConfig, Constant, CommonThd +from cc_debugger.core.enums import NormType, ModeType, FuzzThreshold + + +def load_file(file: Path): + if not file.exists(): + raise FileNotFoundError(f"{file} is not exist") + data = None + if file.suffix == ".pt": + data = torch.load(file) + elif file.suffix == ".p" or file.suffix == ".pickle": + with open(file, 'rb') as f: + data = pickle.load(f) + return data + + +def accept_store_temp(temp_dir, file_list: List) -> List[Path]: + """ + use temp dir to store the upload files + :return: + """ + file_path_list = [] + for index, file in enumerate(file_list): + logger.debug(file) + _file = Path(temp_dir).joinpath(f"{index}_{file.filename}").resolve() + with os.fdopen(os.open(_file, Constant.FILE_FLAGS, Constant.FILE_MODES), 'wb') as fw: + fw.write(file.file.read()) + file_path_list.append(_file) + return file_path_list + + +def store_temp_file(data) -> str: + with tempfile.NamedTemporaryFile(delete=False) as f: + pickle.dump(data, f, protocol=4) + return f.name + + +def convert_tensor_to_device(tensor_obj, device: str = "cpu") -> Any: + """ + if is tensor sequence, convert to device, else return self + :param tensor_obj: + :param device: + :param tensor_type: + :return: + """ + if isinstance(tensor_obj, torch.Tensor): + if device == "cpu" and (tensor_obj.dtype == torch.float16 or tensor_obj.dtype == torch.bfloat16): + return tensor_obj.detach().to(torch.float32).to(device) + return tensor_obj.detach().to(device) + if isinstance(tensor_obj, dict): + return {k: convert_tensor_to_device(v, device) for k, v in tensor_obj.items()} + if isinstance(tensor_obj, (tuple, list)): + return type(tensor_obj)([convert_tensor_to_device(v, device) for v in tensor_obj]) + return tensor_obj + + +def compare_seq(npu_output, other_output) -> bool: + if isinstance(npu_output, torch.Tensor): + cpu_output = other_output.to(npu_output.dtype).to(npu_output.device) + return data_compare_tensor(npu_output.detach(), cpu_output.detach()) + if isinstance(npu_output, dict): + if (len(npu_output) != len(other_output) or + any(not compare_seq(v, other_output.get(k)) for k, v in npu_output.items())): + return False + return True + if isinstance(npu_output, (tuple, list)): + if (len(npu_output) != len(other_output) or + any(not compare_seq(v, other_output[i]) for i, v in enumerate(npu_output))): + return False + return True + if isinstance(npu_output, float): + return math.isclose(npu_output, other_output) + return npu_output == other_output + + +class Tools: + + @staticmethod + def is_add_noise(v: torch.Tensor, func_id: str) -> bool: + """ + 判断是否需要添加扰动 + + :param func_id: torch api 名称 + :param v: 装饰器原输入 + :return: + """ + if v.numel() == 0 or not torch.is_floating_point(v): + logger.warning('[{}] Unsupported tensor types. Cancel adding noise.', func_id) + return False + # 如果inputs的最大值小于对应的精度最小值，则不添加扰动，例如float32下，tensor中最大值若小于1e-8，则不添加扰动 + abs_tol = ThdConfig.ABS_TOL_VALUE_DICT.get(v.dtype) if hasattr(v, + 'dtype') else ThdConfig.NOISE_INPUT_LOWER_BOUND + try: + max_val = torchCFunctions.max(torchCFunctions.abs(v)).item() + except Exception as e: + logger.error(e) + max_val = torchCFunctions.max(torchCFunctions.abs(v.to(torch.float32))).item() + if max_val < abs_tol: + logger.warning('[{}] Maximum value is less than the corresponding precision minimum. Cancel fuzz.', func_id) + return False + return True + + @staticmethod + def to_half(v: Tensor, result: Tensor, func_id: str) -> Tensor: + """ + 精度转为half类型 + + :param func_id: + :param v: + :param result: + :return: + """ + if v.dtype == torch.float16: + result = result.half() + logger.debug('[{}] To half.', func_id) + return result + + @staticmethod + def is_float(inputs: torch.Tensor) -> bool: + """ + 判断tensor是否是浮点类型 + + :param inputs: + :return: True: 是浮点类型, False: 不是浮点类型 + """ + logger.debug('Input type: {}.', inputs.dtype) + return torch.is_floating_point(inputs) + + @staticmethod + def is_iterable(v: object) -> bool: + """ + 判断是否是可迭代对象（List, Tuple） + + :param v: + :return: True: 是, False: 否 + """ + logger.debug('Input type: {}', type(v)) + return isinstance(v, (List, Tuple, Dict)) + + @staticmethod + def base_calculate(origin_output, fuzz_output): + """ + 输出对比预处理 + 1、如果输入含有dim属性，则输出是一个values,indices的元组，此时只比较values值 + 2、将原始输出类型转成为扰动后输出类型（主要是提精度时，采用高精度进行对比） + 3、根据dtype获取极小值的阈值abs_tol + :param origin_output: 原始输出 + :param fuzz_output: 扰动后输出 + :return : + """ + # 如果输入含有dim属性，则输出是一个values,indices的元组(torch.return_types)，此时只比较values值 + if isinstance(origin_output, tuple) and hasattr(origin_output, "values") and hasattr(origin_output, "indices"): + origin_output = origin_output.values + fuzz_output = fuzz_output.values + # 根据dtype获取极小值的阈值abs_tol + if hasattr(origin_output, 'dtype'): + abs_tol = ThdConfig.ABS_TOL_VALUE_DICT.get(fuzz_output.dtype) + else: + abs_tol = FuzzThreshold.F32_THD.value + return origin_output.to(fuzz_output.dtype), fuzz_output, abs_tol + + @staticmethod + def get_ratio_from_specific_norm(origin_output, fuzz_output, norm_type, abs_tol): + """ + 根据指定范数获取对应的ratio + :param origin_output: 原始输出 + :param fuzz_output: 扰动后输出 + :param norm_type: 范数类型 + :param abs_tol: 判定极小值的阈值 + """ + if norm_type == NormType.ENDLESS_NORM: + return Tools.get_endless_norm(origin_output, fuzz_output, abs_tol) + return ThdConfig.COMP_CONSISTENT + + @staticmethod + def convert_overflow_ratio_to_consistent(ratio): + if math.isnan(ratio) or math.isinf(ratio): + return ThdConfig.COMP_CONSISTENT + return ratio + + @staticmethod + def get_endless_norm(origin_output, fuzz_output, abs_tol): + """ + 求origin_output和fuzz_output的第三范数 + :param origin_output: 原始输出 + :param fuzz_output: 扰动后输出 + :param abs_tol: 判断极小值的阈值 + """ + m = torchCFunctions.ones(origin_output.shape).to(dtype=origin_output.dtype).to(origin_output.device) + try: + ratio_tensor1 = torchCFunctions.where(torchCFunctions.abs(fuzz_output) > abs_tol, + torchCFunctions.div(origin_output, fuzz_output), m) + ratio_tensor2 = torchCFunctions.where(torchCFunctions.abs(origin_output) > abs_tol, + torchCFunctions.div(fuzz_output, origin_output), m) + except Exception as e: + logger.error(e) + ratio_tensor1 = torchCFunctions.where(torchCFunctions.abs(fuzz_output.to(torch.float32)) > abs_tol, + torchCFunctions.div(origin_output.to(torch.float32), + fuzz_output.to(torch.float32)), m) + ratio_tensor2 = torchCFunctions.where(torchCFunctions.abs(origin_output.to(torch.float32)) > abs_tol, + torchCFunctions.div(fuzz_output.to(torch.float32), + origin_output.to(torch.float32)), m) + norm1 = Tools.convert_overflow_ratio_to_consistent(torchCFunctions.max(ratio_tensor1).item()) + norm2 = Tools.convert_overflow_ratio_to_consistent(torchCFunctions.max(ratio_tensor2).item()) + norm3 = Tools.convert_overflow_ratio_to_consistent(torchCFunctions.min(ratio_tensor1).item()) + logger.debug("norm1: {norm1}, norm2: {norm2}, norm3: {norm3}", norm1, norm2, norm3) + if norm3 < 0: + # 符号翻转 + ratio = ThdConfig.SYMBOL_FLIPPING + else: + ratio = max(norm1, norm2) + return ratio + + @staticmethod + def ratio_calculate(origin_output, fuzz_output, norm_type: NormType) -> float: + """ + ratio计算 + + :param origin_output: + :param fuzz_output: + :param norm_type: 范数类型 + :return: tau数据值 + """ + try: + origin_output, fuzz_output, abs_tol = Tools.base_calculate(origin_output, fuzz_output) + except Exception as e: + logger.error('When computing ratio, y1 or y2 dtype is not supported {}', e) + return ThdConfig.COMP_CONSISTENT + # 减少运算次数 + # 作小调整，abs开根号 + if base_settings.mode == ModeType.FORWARD.value: + abs_tol = abs_tol ** 0.5 + else: + abs_tol = ThdConfig.BACKWARD_OUTPUT_LOWER_BOUND + return Tools.get_ratio_from_specific_norm(origin_output, fuzz_output, norm_type, abs_tol) + + @staticmethod + def all_ratio_calculate(origin_output, fuzz_output) -> list: + """ + 计算2个ratio + :param origin_output: + :param fuzz_output: + :return: + """ + if isinstance(origin_output, torch.Tensor): + return [Tools.ratio_calculate(origin_output, fuzz_output, norm_type=NormType.ENDLESS_NORM)] + if isinstance(origin_output, (list, tuple)): + ratios = list() + for i, origin_item in enumerate(origin_output): + if isinstance(origin_item, torch.Tensor): + ratio = Tools.ratio_calculate(origin_item, fuzz_output[i], norm_type=NormType.ENDLESS_NORM) + else: + ratio = origin_item / fuzz_output[i] + ratios.append(ratio) + return ratios + return [origin_output / fuzz_output] + + @staticmethod + def is_stack_full(func_id: str) -> bool: + """ + 对装饰器递归深度加上限制 + + :return: + """ + # 判断是否开启递归fuzz扰动 + if base_settings.recursive_fuzz: + return False + is_over = Tools.is_stack_over_max_repeats(CommonThd.STACK_MAX_REPEATS, Constant.WRAPPER_NAME) + + return is_over + + @staticmethod + def is_stack_over_max_repeats(max_repeats: int = 1, func_name: str = Constant.WRAPPER_NAME) -> bool: + nested_depth = max_repeats + frame = sys._getframe(1) + while frame: + if frame.f_code.co_name == func_name: + nested_depth -= 1 + if nested_depth < 0: + return True + frame = frame.f_back + return False + + @staticmethod + def get_first_tensor(tensor_seq: Any): + """ + 返回变量的dtype（如果是张量列表，则返回第一个张量的dtype) + """ + if isinstance(tensor_seq, torch.Tensor): + return tensor_seq + if isinstance(tensor_seq, (list, tuple)): + for i in tensor_seq: + if isinstance(i, torch.Tensor): + return i + log_info = f'This variable is not a tensor sequence: {type(tensor_seq)}' + logger.error(log_info) + raise RuntimeError(log_info) + + @staticmethod + def get_function_names(): + stack = traceback.extract_stack() + function_names = set() + for frame in stack: + function_names.add(frame.name) + return function_names + + @staticmethod + def convert_fuzz_output_to_origin_output(origin_output, fuzzed_output): + """ + if is tensor sequence, convert to device, else return self + :param origin_output: + :param fuzzed_output: + :return: + """ + if isinstance(origin_output, torch.Tensor): + origin_output.data = fuzzed_output.to(origin_output.dtype).to(origin_output.device) + return origin_output + if isinstance(origin_output, dict): + output = dict() + for k, v in origin_output.items(): + output[k] = Tools.convert_fuzz_output_to_origin_output(v, fuzzed_output[k]) + return output + if isinstance(origin_output, (tuple, list)): + result = list() + for i, v in enumerate(origin_output): + result.append(Tools.convert_fuzz_output_to_origin_output(v, fuzzed_output[i])) + return type(origin_output)(result) + return origin_output + + @staticmethod + def check_output_type(output) -> bool: + """ + 校验输出类型 + """ + if isinstance(output, torch.Tensor) and Tools.is_float(output): + return True + if isinstance(output, (list, tuple)): + for i in output: + if isinstance(i, torch.Tensor) and Tools.is_float(i): + return True + return False + + @staticmethod + def write_data_to_csv(result: list, result_header: Tuple, file_name: str): + if len(result) == 0: + return + flags = os.O_WRONLY | os.O_CREAT + modes = Constant.FILE_MODES + is_exists = os.path.exists(file_name) + with os.fdopen(os.open(file_name, flags, modes), 'a+' if is_exists else 'w', newline='') as csvfile: + spawnwriter = csv.writer(csvfile) + if not is_exists: + spawnwriter.writerow(result_header) + spawnwriter.writerows(result) \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/core/__init__.py new file mode 100644 index 000000000..c39257b25 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/config.py b/debug/accuracy_tools/CCDebugger/cc_debugger/core/config.py new file mode 100644 index 000000000..d15dc9702 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/config.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import os +from pathlib import Path +from typing import List +from loguru import logger + +from cc_debugger.core.enums import CCDMode +from cc_debugger.core.settings.base import NPUEnv, GPUEnv, BaseEnv + + +class CCDConfig: + def __init__(self): + # 是否运行，指的是进行插桩后是否运行提精度、添加扰动等对输入输出进行处理的操作 + # 全局配置`ccd_flag`设置的只是是否进行插桩 + self.is_running: bool = False + # 待测api列表 + self.test_apis: List[str] = [] + self.current_step: int = -1 + self.check_dir: Path = base_settings.root_dir.joinpath('check') + self.fig_dir: Path = base_settings.root_dir.joinpath('fig') + self.statistic_dir: Path = base_settings.root_dir.joinpath('statistic') + self.api_preheat: dict = dict() + self.api_thd: dict = dict() + self.api_compare: dict = dict() + self.preheat_dir: Path = base_settings.root_dir.joinpath('preheat') + + @staticmethod + def configure_logger() -> None: + """ + 配置日志 + + :return: + """ + os_pid = os.getpid() + + def log_format(record): + record['extra']['pid'] = os_pid + format_string = '{time:YYYY-MM-DD HH:mm:ss} | {level} | [{extra[pid]}] | ' \ + '{module}.{function}:{line} - {message}\n' + return format_string.format(**record) + + logger.remove() + logger.add(base_settings.root_dir.joinpath("ccd.log"), level=base_settings.log_level, format=log_format) + + +base_settings = BaseEnv() +device_settings = { + CCDMode.NPU: NPUEnv(), + CCDMode.GPU: GPUEnv(), + CCDMode.CPU: base_settings +} +config = CCDConfig() +config.configure_logger() diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/const.py b/debug/accuracy_tools/CCDebugger/cc_debugger/core/const.py new file mode 100644 index 000000000..cbdad79f0 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/const.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +常量类 + +""" +import os +import stat +from typing import Dict +import torch +from cc_debugger.core.enums import FuzzThreshold, CCDMode + + +class FigConfig: + """ + 绘图参数 + y_log 是否绘制 对数纵坐标 + abs_mode 是否修改 结果绝对值 + max 截断上界 + scale 误差缩放系数 + cnt 每单位误差内的组数 + """ + Y_LOG = True + ABS_MODE = False + LOWER_BOUND: float = 1e-5 + MAX = 5 + CNT = 1000 + DTYPE_SCALE_THD = { + torch.bfloat16: 250, + torch.float16: 1e3, + torch.float32: 1e5 + } + + +class Constant: + WRAPPER_NAME = 'fuzz_wrapper' + TORCH_FLOAT_PREFIX = 'torch.' + DEVICE = 'device' + META = 'meta' + FILE_FLAGS = os.O_RDWR | os.O_TRUNC | os.O_CREAT + FILE_MODES = stat.S_IRUSR | stat.S_IWUSR + POOL_TASK_NUM = 64 + PRIORITY = [CCDMode.NPU, CCDMode.GPU, CCDMode.CPU] + UP_INPUT_FILE = "up_input_file" + + +class CommonThd: + # 装饰器在堆栈中的最大重复数量 + STACK_MAX_REPEATS = 2 + + +class ThdConfig: + # fuzz扰动前后输出符号翻转 + SYMBOL_FLIPPING = 8.0 + # 反向下界 + BACKWARD_OUTPUT_LOWER_BOUND = 1e-3 + # 输入噪音下界 + NOISE_INPUT_LOWER_BOUND = 1e-8 + # 比对一致 + COMP_CONSISTENT = 1.0 + # 预热算子初始阈值 + PREHEAT_INITIAL_THD = 2.05 + # 算子阈值更新的步长（* STEP or / STEP) + API_THD_STEP = 2.0 + # 默认的预热阈值下界 + DEFAULT_PRE_THD: float = 1.0002 + # 不同dtype的预热阈值下界 + DTYPE_PRE_THD = { + torch.float16: 1.002, + torch.float32: 1.0002 + } + PERTURBATION_VALUE_DICT: Dict = { + torch.bfloat16: FuzzThreshold.BF16_THD.value, + torch.float16: FuzzThreshold.F16_THD.value, + torch.float32: FuzzThreshold.F32_THD.value, + torch.float64: FuzzThreshold.F64_THD.value + } + ABS_TOL_VALUE_DICT: Dict = { + torch.bfloat16: FuzzThreshold.BF16_THD.value, + torch.float16: FuzzThreshold.F16_THD.value, + torch.float32: FuzzThreshold.F32_THD.value, + torch.float64: FuzzThreshold.F64_THD.value + } + diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/enums.py b/debug/accuracy_tools/CCDebugger/cc_debugger/core/enums.py new file mode 100644 index 000000000..58bea5c01 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/enums.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +枚举类 + +""" +from enum import Enum + + +class TensorFloatType(Enum): + B_FLOAT_16 = 'bfloat16' + FLOAT_16 = 'float16' + FLOAT_32 = 'float32' + FLOAT_64 = 'float64' + + +class PerturbationModeType(Enum): + """ + 扰动方式类型 + + """ + ADD_NOISE = 'add_noise' + CHANGE_VALUE = 'change_value' + IMPROVE_PRECISION = 'improve_precision' + NO_CHANGE = "no_change" + + +class NormType(Enum): + """ + 范数类型 + """ + ONE_NORM = (1, 'one_norm') + TWO_NORM = (2, 'two_norm') + ENDLESS_NORM = (3, 'endless_norm') + + +class HandlerType(Enum): + """ + Handler的类型定义 + + """ + CHECK = 'check' + FIX = 'fix' + PREHEAT = "preheat" + PFIX = 'pfix' + RELATIVE = 'relative' + + +class FuzzThreshold(Enum): + """ + 扰动阈值定义 + """ + BF16_THD = 1e-4 + F16_THD = 1e-6 + F32_THD = 1e-8 + F64_THD = 1e-16 + + +class ModeType(Enum): + """ + fuzz扰动模式(forward、backward) + """ + FORWARD = 'forward' + BACKWARD = 'backward' + + +class CCDMode(Enum): + """ + 工具比对模式 + """ + CPU = 'cpu' + NPU = 'npu' + GPU = 'gpu' + + +class PostResField(Enum): + ERROR = "error" + RESULT = "result" + + +class InputKey(Enum): + ARGS = "call_param_args" + KWARGS = "call_param_kwargs" + CLA = "init_class" + GRAD = "grad_outputs" + NEED_GRAD = "needs_input_grad" + OUTPUT = "output" + + +class Apis(Enum): + API_RUN_COMPARE = "run-api-compare" + API_AUTOGRAD = "api-autograd" + API_RESULT = "run-api-result" + MODEL_RESULT = "run-model-result" + API_SINGLE_COMPARE = "single_compare" + SINGLE_PERF = "single_perf" diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/exceptions.py b/debug/accuracy_tools/CCDebugger/cc_debugger/core/exceptions.py new file mode 100644 index 000000000..cefb9d5c7 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/exceptions.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +class UnsupportedTypeException(Exception): + """ + 类型不支持 异常 + + """ + + def __init__(self): + super().__init__('Unsupported type.') + + +class OverflowException(Exception): + def __init__(self): + super().__init__('The operation result overflows.') diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/__init__.py new file mode 100644 index 000000000..601aa4d99 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/__init__.py @@ -0,0 +1,3 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.env b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.env new file mode 100644 index 000000000..d05e34564 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.env @@ -0,0 +1,24 @@ +# 是否开启CCD +ccd_flag=False +# 输出存储路径 +root_dir=./ccd_output +# 日志级别 +log_level=INFO +# ccd对比机器，目前支持[npu, cpu, gpu]，可多选,设置不同对比方式的step间隔 +ccd_mode='{"npu": 0}' +# 前向或者反向 +mode=forward +# 是否记录模型所有调用的api +record_flag=False +# 输出处理模式 +handler_type=check +# 是否开启范围插装 +range_fuzz=False +# 开始step +step_start=0 +# 结束step +step_end=3 +# 是否递归插装 +recursive_fuzz=False +# 是否step单算子插装 +single_fuzz=False diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.py b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.py new file mode 100644 index 000000000..fa81f5013 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/base.py @@ -0,0 +1,48 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from pathlib import Path +from typing import Dict +from pydantic_settings import BaseSettings +from cc_debugger.core.enums import PerturbationModeType, HandlerType, ModeType, CCDMode + + +class BaseEnv(BaseSettings): + class Config: + env_file = Path(__file__).parent.joinpath("base.env") + validate_assignment = True + + # 是否开启ccd + ccd_flag: bool = False + # 输出结果路径 + root_dir: Path = Path("./ccd_output") + log_level: str = "INFO" + # 比对模式 + ccd_mode: Dict[CCDMode, int] = {CCDMode.NPU: 0} + mode: ModeType = ModeType.FORWARD + record_flag: bool = False + handler_type: HandlerType = HandlerType.CHECK + range_fuzz: bool = False + step_start: int = 0 + step_end: int = 9 + recursive_fuzz: bool = False + single_fuzz: bool = False + + +class NPUEnv(BaseSettings): + class Config: + env_file = Path(__file__).parent.joinpath("npu.env") + validate_assignment = True + + pert_type: PerturbationModeType = PerturbationModeType.IMPROVE_PRECISION + is_preheat: bool = True + max_sample: int = 20 + preheat_step: int = 15 + + +class GPUEnv(BaseSettings): + class Config: + env_file = Path(__file__).parent.joinpath("gpu.env") + validate_assignment = True + + server_url: str = "http://7.216.52.7:8081" diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/gpu.env b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/gpu.env new file mode 100644 index 000000000..a5660b7a6 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/gpu.env @@ -0,0 +1,2 @@ +# gpu服务地址 +server_url=http://7.216.52.7:8081 \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/npu.env b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/npu.env new file mode 100644 index 000000000..e7b78b170 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/core/settings/npu.env @@ -0,0 +1,8 @@ +# 输入扰动模式 +pert_type=improve_precision +# 是否开启预热 +is_preheat=True +# 最大采样数 +max_sample=20 +# 预热step +preheat_step=15 \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api.txt b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api.txt new file mode 100644 index 000000000..1fe87ce85 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api.txt @@ -0,0 +1,25 @@ +torch.cos +torch.mul +torch.nn.functional.cross_entropy +torch.nn.functional.dropout +torch.nn.functional.embedding +torch.nn.functional.linear +torch.nn.functional.scaled_dot_product_attention +torch.nn.functional.silu +torch.norm +torch.pow +torch.randperm +torch.rsqrt +torch.sin +torch.Tensor.__iadd__ +torch.Tensor.__isub__ +torch.Tensor.mean +torch.Tensor.__mul__ +torch.Tensor.mul_ +torch.Tensor.norm +torch.Tensor.permute +torch.Tensor.reciprocal +torch.Tensor.split +torch.Tensor.sqrt +torch.Tensor.__sub__ +torch.Tensor.sum \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_all.txt b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_all.txt new file mode 100644 index 000000000..69bcdc28b --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_all.txt @@ -0,0 +1,1647 @@ +torch.zeros +torch.zeros_like +torch.ones +torch.ones_like +torch.arange +torch.range +torch.linspace +torch.logspace +torch.eye +torch.empty_like +torch.empty_strided +torch.full +torch.full_like +torch.quantize_per_tensor +torch.quantize_per_channel +torch.dequantize +torch.complex +torch.polar +torch.heaviside +torch.adjoint +torch.argwhere +torch.cat +torch.concat +torch.conj +torch.chunk +torch.dsplit +torch.column_stack +torch.dstack +torch.gather +torch.hsplit +torch.hstack +torch.index_add +torch.index_select +torch.masked_select +torch.movedim +torch.narrow +torch.nonzero +torch.permute +torch.reshape +torch.row_stack +torch.select +torch.scatter +torch.diagonal_scatter +torch.select_scatter +torch.slice_scatter +torch.scatter_add +torch.scatter_reduce +torch.split +torch.squeeze +torch.stack +torch.swapaxes +torch.swapdims +torch.take +torch.take_along_dim +torch.tensor_split +torch.tile +torch.transpose +torch.unbind +torch.unsqueeze +torch.vsplit +torch.vstack +torch.where +torch.bernoulli +torch.multinomial +torch.normal +torch.poisson +torch.rand +torch.rand_like +torch.randint +torch.randint_like +torch.randn +torch.randn_like +torch.randperm +torch.quasirandom.SobolEngine +torch.quasirandom.SobolEngine.draw +torch.quasirandom.SobolEngine.draw_base2 +torch.quasirandom.SobolEngine.fast_forward +torch.quasirandom.SobolEngine.reset +torch.abs +torch.absolute +torch.acos +torch.arccos +torch.acosh +torch.arccosh +torch.add +torch.addcdiv +torch.addcmul +torch.angle +torch.asin +torch.arcsin +torch.asinh +torch.arcsinh +torch.atan +torch.arctan +torch.atanh +torch.arctanh +torch.atan2 +torch.arctan2 +torch.bitwise_not +torch.bitwise_and +torch.bitwise_or +torch.bitwise_xor +torch.bitwise_left_shift +torch.bitwise_right_shift +torch.ceil +torch.clamp +torch.clip +torch.conj_physical +torch.copysign +torch.cos +torch.cosh +torch.deg2rad +torch.div +torch.divide +torch.digamma +torch.erf +torch.erfc +torch.erfinv +torch.exp +torch.exp2 +torch.expm1 +torch.fake_quantize_per_channel_affine +torch.fake_quantize_per_tensor_affine +torch.fix +torch.float_power +torch.floor +torch.floor_divide +torch.fmod +torch.frac +torch.frexp +torch.gradient +torch.imag +torch.ldexp +torch.lerp +torch.lgamma +torch.log +torch.log10 +torch.log1p +torch.log2 +torch.logaddexp +torch.logaddexp2 +torch.logical_and +torch.logical_not +torch.logical_or +torch.logical_xor +torch.logit +torch.hypot +torch.i0 +torch.igamma +torch.igammac +torch.mul +torch.multiply +torch.mvlgamma +torch.nan_to_num +torch.neg +torch.negative +torch.nextafter +torch.polygamma +torch.positive +torch.pow +torch.quantized_batch_norm +torch.quantized_max_pool1d +torch.quantized_max_pool2d +torch.rad2deg +torch.real +torch.reciprocal +torch.remainder +torch.round +torch.rsqrt +torch.sigmoid +torch.sign +torch.sgn +torch.signbit +torch.sin +torch.sinc +torch.sinh +torch.sqrt +torch.square +torch.sub +torch.subtract +torch.tan +torch.tanh +torch.true_divide +torch.trunc +torch.xlogy +torch.argmax +torch.argmin +torch.amax +torch.amin +torch.aminmax +torch.all +torch.any +torch.max +torch.max +torch.max +torch.min +torch.min +torch.min +torch.dist +torch.logsumexp +torch.mean +torch.mean +torch.nanmean +torch.median +torch.nanmedian +torch.mode +torch.norm +torch.nansum +torch.prod +torch.quantile +torch.nanquantile +torch.std +torch.std +torch.std_mean +torch.sum +torch.unique +torch.unique_consecutive +torch.var +torch.var_mean +torch.count_nonzero +torch.allclose +torch.argsort +torch.eq +torch.equal +torch.ge +torch.greater_equal +torch.gt +torch.greater +torch.isclose +torch.isfinite +torch.isin +torch.isinf +torch.isposinf +torch.isneginf +torch.isnan +torch.isreal +torch.kthvalue +torch.le +torch.less_equal +torch.lt +torch.less +torch.maximum +torch.minimum +torch.fmax +torch.fmin +torch.ne +torch.not_equal +torch.sort +torch.topk +torch.msort +torch.stft +torch.istft +torch.bartlett_window +torch.blackman_window +torch.hamming_window +torch.hann_window +torch.kaiser_window +torch.atleast_1d +torch.atleast_2d +torch.atleast_3d +torch.bincount +torch.block_diag +torch.broadcast_tensors +torch.broadcast_to +torch.broadcast_shapes +torch.bucketize +torch.cartesian_prod +torch.cdist +torch.clone +torch.combinations +torch.corrcoef +torch.cov +torch.cross +torch.cummax +torch.cummin +torch.cumprod +torch.cumsum +torch.diag +torch.diag_embed +torch.diagflat +torch.diagonal +torch.diff +torch.einsum +torch.flatten +torch.flip +torch.fliplr +torch.flipud +torch.kron +torch.rot90 +torch.gcd +torch.histc +torch.histogram +torch.histogramdd +torch.meshgrid +torch.lcm +torch.logcumsumexp +torch.ravel +torch.renorm +torch.repeat_interleave +torch.roll +torch.searchsorted +torch.tensordot +torch.trace +torch.tril +torch.tril_indices +torch.triu +torch.triu_indices +torch.vander +torch.view_as_real +torch.view_as_complex +torch.resolve_conj +torch.resolve_neg +torch.addbmm +torch.addmm +torch.addmv +torch.addr +torch.baddbmm +torch.bmm +torch.chain_matmul +torch.cholesky +torch.cholesky_inverse +torch.cholesky_solve +torch.dot +torch.eig +torch.geqrf +torch.ger +torch.inner +torch.inverse +torch.det +torch.logdet +torch.slogdet +torch.lstsq +torch.lu +torch.lu_solve +torch.lu_unpack +torch.matmul +torch.matrix_power +torch.matrix_rank +torch.matrix_exp +torch.mm +torch.mv +torch.orgqr +torch.ormqr +torch.outer +torch.pinverse +torch.qr +torch.solve +torch.svd +torch.svd_lowrank +torch.pca_lowrank +torch.symeig +torch.lobpcg +torch.trapz +torch.trapezoid +torch.cumulative_trapezoid +torch.triangular_solve +torch.vdot +torch.nn.Conv1d +torch.nn.Conv2d +torch.nn.Conv3d +torch.nn.ConvTranspose1d +torch.nn.ConvTranspose2d +torch.nn.ConvTranspose3d +torch.nn.LazyConv1d +torch.nn.LazyConv2d +torch.nn.LazyConv3d +torch.nn.LazyConvTranspose1d +torch.nn.LazyConvTranspose2d +torch.nn.LazyConvTranspose3d +torch.nn.Unfold +torch.nn.Fold +torch.nn.MaxPool1d +torch.nn.MaxPool2d +torch.nn.MaxPool3d +torch.nn.MaxUnpool1d +torch.nn.MaxUnpool2d +torch.nn.MaxUnpool3d +torch.nn.AvgPool1d +torch.nn.AvgPool2d +torch.nn.AvgPool3d +torch.nn.FractionalMaxPool2d +torch.nn.FractionalMaxPool3d +torch.nn.LPPool1d +torch.nn.LPPool2d +torch.nn.AdaptiveMaxPool1d +torch.nn.AdaptiveMaxPool2d +torch.nn.AdaptiveMaxPool3d +torch.nn.AdaptiveAvgPool1d +torch.nn.AdaptiveAvgPool2d +torch.nn.AdaptiveAvgPool3d +torch.nn.ReflectionPad1d +torch.nn.ReflectionPad2d +torch.nn.ReflectionPad3d +torch.nn.ReplicationPad1d +torch.nn.ReplicationPad2d +torch.nn.ReplicationPad3d +torch.nn.ZeroPad2d +torch.nn.ConstantPad1d +torch.nn.ConstantPad2d +torch.nn.ConstantPad3d +torch.nn.ELU +torch.nn.Hardshrink +torch.nn.Hardsigmoid +torch.nn.Hardtanh +torch.nn.Hardswish +torch.nn.LeakyReLU +torch.nn.LogSigmoid +torch.nn.MultiheadAttention +torch.nn.MultiheadAttention.forward +torch.nn.PReLU +torch.nn.ReLU +torch.nn.ReLU6 +torch.nn.RReLU +torch.nn.SELU +torch.nn.CELU +torch.nn.GELU +torch.nn.Sigmoid +torch.nn.SiLU +torch.nn.Mish +torch.nn.Softplus +torch.nn.Softshrink +torch.nn.Softsign +torch.nn.Tanh +torch.nn.Tanhshrink +torch.nn.Threshold +torch.nn.GLU +torch.nn.Softmin +torch.nn.Softmax +torch.nn.Softmax2d +torch.nn.LogSoftmax +torch.nn.AdaptiveLogSoftmaxWithLoss +torch.nn.AdaptiveLogSoftmaxWithLoss.log_prob +torch.nn.AdaptiveLogSoftmaxWithLoss.predict +torch.nn.BatchNorm1d +torch.nn.BatchNorm2d +torch.nn.BatchNorm3d +torch.nn.LazyBatchNorm1d +torch.nn.LazyBatchNorm2d +torch.nn.LazyBatchNorm3d +torch.nn.GroupNorm +torch.nn.SyncBatchNorm +torch.nn.InstanceNorm1d +torch.nn.InstanceNorm2d +torch.nn.InstanceNorm3d +torch.nn.LazyInstanceNorm1d +torch.nn.LazyInstanceNorm2d +torch.nn.LazyInstanceNorm3d +torch.nn.LayerNorm +torch.nn.LocalResponseNorm +torch.nn.RNNBase +torch.nn.RNNBase.flatten_parameters +torch.nn.RNN +torch.nn.LSTM +torch.nn.GRU +torch.nn.RNNCell +torch.nn.LSTMCell +torch.nn.GRUCell +torch.nn.Transformer +torch.nn.Transformer.forward +torch.nn.Transformer.generate_square_subsequent_mask +torch.nn.TransformerEncoder +torch.nn.TransformerEncoder.forward +torch.nn.TransformerDecoder +torch.nn.TransformerDecoder.forward +torch.nn.TransformerEncoderLayer +torch.nn.TransformerEncoderLayer.forward +torch.nn.TransformerDecoderLayer +torch.nn.TransformerDecoderLayer.forward +torch.nn.utils.clip_grad_norm_ +torch.nn.utils.clip_grad_value_ +torch.nn.utils.parameters_to_vector +torch.nn.utils.vector_to_parameters +torch.nn.utils.weight_norm +torch.nn.utils.remove_weight_norm +torch.nn.utils.spectral_norm +torch.nn.utils.remove_spectral_norm +torch.nn.utils.skip_init +torch.nn.utils.parametrizations.orthogonal +torch.nn.utils.parametrizations.spectral_norm +torch.nn.utils.parametrize.register_parametrization +torch.nn.utils.parametrize.remove_parametrizations +torch.nn.utils.parametrize.cached +torch.nn.utils.parametrize.is_parametrized +torch.nn.utils.rnn.pack_padded_sequence +torch.nn.utils.rnn.pad_packed_sequence +torch.nn.utils.rnn.pad_sequence +torch.nn.utils.rnn.pack_sequence +torch.nn.functional.conv1d +torch.nn.functional.conv2d +torch.nn.functional.conv3d +torch.nn.functional.conv_transpose1d +torch.nn.functional.conv_transpose2d +torch.nn.functional.conv_transpose3d +torch.nn.functional.unfold +torch.nn.functional.fold +torch.nn.functional.avg_pool1d +torch.nn.functional.avg_pool2d +torch.nn.functional.avg_pool3d +torch.nn.functional.max_pool1d +torch.nn.functional.max_pool2d +torch.nn.functional.max_pool3d +torch.nn.functional.max_unpool1d +torch.nn.functional.max_unpool2d +torch.nn.functional.max_unpool3d +torch.nn.functional.lp_pool1d +torch.nn.functional.lp_pool2d +torch.nn.functional.adaptive_max_pool1d +torch.nn.functional.adaptive_max_pool2d +torch.nn.functional.adaptive_max_pool3d +torch.nn.functional.adaptive_avg_pool1d +torch.nn.functional.adaptive_avg_pool2d +torch.nn.functional.adaptive_avg_pool3d +torch.nn.functional.fractional_max_pool2d +torch.nn.functional.fractional_max_pool3d +torch.nn.functional.threshold +torch.nn.functional.threshold_ +torch.nn.functional.relu +torch.nn.functional.relu_ +torch.nn.functional.hardtanh +torch.nn.functional.hardtanh_ +torch.nn.functional.hardswish +torch.nn.functional.relu6 +torch.nn.functional.elu +torch.nn.functional.elu_ +torch.nn.functional.selu +torch.nn.functional.celu +torch.nn.functional.leaky_relu +torch.nn.functional.leaky_relu_ +torch.nn.functional.prelu +torch.nn.functional.rrelu +torch.nn.functional.rrelu_ +torch.nn.functional.glu +torch.nn.functional.gelu +torch.nn.functional.logsigmoid +torch.nn.functional.hardshrink +torch.nn.functional.tanhshrink +torch.nn.functional.softsign +torch.nn.functional.softplus +torch.nn.functional.softmin +torch.nn.functional.softmax +torch.nn.functional.softshrink +torch.nn.functional.gumbel_softmax +torch.nn.functional.log_softmax +torch.nn.functional.tanh +torch.nn.functional.sigmoid +torch.nn.functional.hardsigmoid +torch.nn.functional.silu +torch.nn.functional.mish +torch.nn.functional.batch_norm +torch.nn.functional.group_norm +torch.nn.functional.instance_norm +torch.nn.functional.layer_norm +torch.nn.functional.local_response_norm +torch.nn.functional.normalize +torch.nn.functional.linear +torch.nn.functional.bilinear +torch.nn.functional.dropout +torch.nn.functional.alpha_dropout +torch.nn.functional.feature_alpha_dropout +torch.nn.functional.dropout2d +torch.nn.functional.dropout3d +torch.nn.functional.embedding +torch.nn.functional.embedding_bag +torch.nn.functional.one_hot +torch.nn.functional.pairwise_distance +torch.nn.functional.cosine_similarity +torch.nn.functional.pdist +torch.nn.functional.binary_cross_entropy +torch.nn.functional.binary_cross_entropy_with_logits +torch.nn.functional.poisson_nll_loss +torch.nn.functional.cosine_embedding_loss +torch.nn.functional.cross_entropy +torch.nn.functional.ctc_loss +torch.nn.functional.gaussian_nll_loss +torch.nn.functional.hinge_embedding_loss +torch.nn.functional.kl_div +torch.nn.functional.l1_loss +torch.nn.functional.mse_loss +torch.nn.functional.margin_ranking_loss +torch.nn.functional.multilabel_margin_loss +torch.nn.functional.multilabel_soft_margin_loss +torch.nn.functional.multi_margin_loss +torch.nn.functional.nll_loss +torch.nn.functional.huber_loss +torch.nn.functional.smooth_l1_loss +torch.nn.functional.soft_margin_loss +torch.nn.functional.triplet_margin_loss +torch.nn.functional.triplet_margin_with_distance_loss +torch.nn.functional.pixel_shuffle +torch.nn.functional.pixel_unshuffle +torch.nn.functional.pad +torch.nn.functional.interpolate +torch.nn.functional.upsample +torch.nn.functional.upsample_nearest +torch.nn.functional.upsample_bilinear +torch.nn.functional.grid_sample +torch.nn.functional.affine_grid +torch.Tensor.abs +torch.Tensor.abs_ +torch.Tensor.absolute +torch.Tensor.absolute_ +torch.Tensor.acos +torch.Tensor.acos_ +torch.Tensor.arccos +torch.Tensor.arccos_ +torch.Tensor.add +torch.Tensor.add_ +torch.Tensor.addbmm +torch.Tensor.addbmm_ +torch.Tensor.addcdiv +torch.Tensor.addcdiv_ +torch.Tensor.addcmul +torch.Tensor.addcmul_ +torch.Tensor.addmm +torch.Tensor.addmm_ +torch.Tensor.sspaddmm +torch.Tensor.addmv +torch.Tensor.addmv_ +torch.Tensor.addr +torch.Tensor.addr_ +torch.Tensor.adjoint +torch.Tensor.allclose +torch.Tensor.amax +torch.Tensor.amin +torch.Tensor.aminmax +torch.Tensor.angle +torch.Tensor.apply_ +torch.Tensor.argmax +torch.Tensor.argmin +torch.Tensor.argsort +torch.Tensor.argwhere +torch.Tensor.asin +torch.Tensor.asin_ +torch.Tensor.arcsin +torch.Tensor.arcsin_ +torch.Tensor.as_strided +torch.Tensor.atan +torch.Tensor.atan_ +torch.Tensor.arctan +torch.Tensor.arctan_ +torch.Tensor.atan2 +torch.Tensor.atan2_ +torch.Tensor.arctan2 +torch.Tensor.arctan2_ +torch.Tensor.all +torch.Tensor.any +torch.Tensor.baddbmm +torch.Tensor.baddbmm_ +torch.Tensor.bernoulli +torch.Tensor.bernoulli_ +torch.Tensor.bfloat16 +torch.Tensor.bincount +torch.Tensor.bitwise_not +torch.Tensor.bitwise_not_ +torch.Tensor.bitwise_and +torch.Tensor.bitwise_and_ +torch.Tensor.bitwise_or +torch.Tensor.bitwise_or_ +torch.Tensor.bitwise_xor +torch.Tensor.bitwise_xor_ +torch.Tensor.bitwise_left_shift +torch.Tensor.bitwise_left_shift_ +torch.Tensor.bitwise_right_shift +torch.Tensor.bitwise_right_shift_ +torch.Tensor.bmm +torch.Tensor.bool +torch.Tensor.byte +torch.Tensor.broadcast_to +torch.Tensor.cauchy_ +torch.Tensor.ceil +torch.Tensor.ceil_ +torch.Tensor.char +torch.Tensor.cholesky +torch.Tensor.cholesky_inverse +torch.Tensor.cholesky_solve +torch.Tensor.chunk +torch.Tensor.clamp +torch.Tensor.clamp_ +torch.Tensor.clip +torch.Tensor.clip_ +torch.Tensor.clone +torch.Tensor.contiguous +torch.Tensor.copy_ +torch.Tensor.conj +torch.Tensor.conj_physical +torch.Tensor.conj_physical_ +torch.Tensor.resolve_conj +torch.Tensor.resolve_neg +torch.Tensor.copysign +torch.Tensor.copysign_ +torch.Tensor.cos +torch.Tensor.cos_ +torch.Tensor.cosh +torch.Tensor.cosh_ +torch.Tensor.corrcoef +torch.Tensor.count_nonzero +torch.Tensor.cov +torch.Tensor.acosh +torch.Tensor.acosh_ +torch.Tensor.arccosh +torch.Tensor.arccosh_ +torch.Tensor.cpu +torch.Tensor.cross +torch.Tensor.cuda +torch.Tensor.logcumsumexp +torch.Tensor.cummax +torch.Tensor.cummin +torch.Tensor.cumprod +torch.Tensor.cumprod_ +torch.Tensor.cumsum +torch.Tensor.cumsum_ +torch.Tensor.deg2rad +torch.Tensor.dequantize +torch.Tensor.det +torch.Tensor.dense_dim +torch.Tensor.diag +torch.Tensor.diag_embed +torch.Tensor.diagflat +torch.Tensor.diagonal +torch.Tensor.diagonal_scatter +torch.Tensor.fill_diagonal_ +torch.Tensor.fmax +torch.Tensor.fmin +torch.Tensor.diff +torch.Tensor.digamma +torch.Tensor.digamma_ +torch.Tensor.dist +torch.Tensor.div +torch.Tensor.div_ +torch.Tensor.divide +torch.Tensor.divide_ +torch.Tensor.dot +torch.Tensor.double +torch.Tensor.eig +torch.Tensor.eq +torch.Tensor.eq_ +torch.Tensor.equal +torch.Tensor.erf +torch.Tensor.erf_ +torch.Tensor.erfc +torch.Tensor.erfc_ +torch.Tensor.erfinv +torch.Tensor.erfinv_ +torch.Tensor.exp +torch.Tensor.exp_ +torch.Tensor.expm1 +torch.Tensor.expm1_ +torch.Tensor.expand +torch.Tensor.expand_as +torch.Tensor.exponential_ +torch.Tensor.fix +torch.Tensor.fix_ +torch.Tensor.fill_ +torch.Tensor.flip +torch.Tensor.fliplr +torch.Tensor.flipud +torch.Tensor.float +torch.Tensor.float_power +torch.Tensor.float_power_ +torch.Tensor.floor +torch.Tensor.floor_ +torch.Tensor.floor_divide +torch.Tensor.floor_divide_ +torch.Tensor.fmod +torch.Tensor.fmod_ +torch.Tensor.frac +torch.Tensor.frac_ +torch.Tensor.frexp +torch.Tensor.gather +torch.Tensor.gcd +torch.Tensor.gcd_ +torch.Tensor.ge +torch.Tensor.ge_ +torch.Tensor.greater_equal +torch.Tensor.greater_equal_ +torch.Tensor.geometric_ +torch.Tensor.geqrf +torch.Tensor.ger +torch.Tensor.gt +torch.Tensor.gt_ +torch.Tensor.greater +torch.Tensor.greater_ +torch.Tensor.half +torch.Tensor.hardshrink +torch.Tensor.heaviside +torch.Tensor.histc +torch.Tensor.histogram +torch.Tensor.hsplit +torch.Tensor.hypot +torch.Tensor.hypot_ +torch.Tensor.i0 +torch.Tensor.i0_ +torch.Tensor.igamma +torch.Tensor.igamma_ +torch.Tensor.igammac +torch.Tensor.igammac_ +torch.Tensor.index_add_ +torch.Tensor.index_add +torch.Tensor.index_copy_ +torch.Tensor.index_copy +torch.Tensor.index_fill_ +torch.Tensor.index_fill +torch.Tensor.index_put_ +torch.Tensor.index_put +torch.Tensor.index_select +torch.Tensor.inner +torch.Tensor.int +torch.Tensor.int_repr +torch.Tensor.inverse +torch.Tensor.isclose +torch.Tensor.isfinite +torch.Tensor.isinf +torch.Tensor.isposinf +torch.Tensor.isneginf +torch.Tensor.is_complex +torch.Tensor.is_conj +torch.Tensor.istft +torch.Tensor.isreal +torch.Tensor.kthvalue +torch.Tensor.lcm +torch.Tensor.lcm_ +torch.Tensor.ldexp +torch.Tensor.ldexp_ +torch.Tensor.le +torch.Tensor.le_ +torch.Tensor.less_equal +torch.Tensor.less_equal_ +torch.Tensor.lerp +torch.Tensor.lerp_ +torch.Tensor.lgamma +torch.Tensor.lgamma_ +torch.Tensor.log +torch.Tensor.log_ +torch.Tensor.logdet +torch.Tensor.log10 +torch.Tensor.log10_ +torch.Tensor.log1p +torch.Tensor.log1p_ +torch.Tensor.log2 +torch.Tensor.log2_ +torch.Tensor.log_normal_ +torch.Tensor.logaddexp +torch.Tensor.logaddexp2 +torch.Tensor.logsumexp +torch.Tensor.logical_and +torch.Tensor.logical_and_ +torch.Tensor.logical_not +torch.Tensor.logical_not_ +torch.Tensor.logical_or +torch.Tensor.logical_or_ +torch.Tensor.logical_xor +torch.Tensor.logical_xor_ +torch.Tensor.logit +torch.Tensor.logit_ +torch.Tensor.long +torch.Tensor.lstsq +torch.Tensor.lt +torch.Tensor.lt_ +torch.Tensor.less +torch.Tensor.less_ +torch.Tensor.lu +torch.Tensor.lu_solve +torch.Tensor.as_subclass +torch.Tensor.masked_scatter_ +torch.Tensor.masked_scatter +torch.Tensor.masked_fill_ +torch.Tensor.masked_fill +torch.Tensor.masked_select +torch.Tensor.matmul +torch.Tensor.matrix_power +torch.Tensor.matrix_exp +torch.Tensor.max +torch.Tensor.maximum +torch.Tensor.mean +torch.Tensor.nanmean +torch.Tensor.median +torch.Tensor.nanmedian +torch.Tensor.min +torch.Tensor.minimum +torch.Tensor.mm +torch.Tensor.smm +torch.Tensor.mode +torch.Tensor.movedim +torch.Tensor.moveaxis +torch.Tensor.msort +torch.Tensor.mul +torch.Tensor.mul_ +torch.Tensor.multiply +torch.Tensor.multiply_ +torch.Tensor.multinomial +torch.Tensor.mv +torch.Tensor.mvlgamma +torch.Tensor.mvlgamma_ +torch.Tensor.nansum +torch.Tensor.narrow +torch.Tensor.narrow_copy +torch.Tensor.nan_to_num +torch.Tensor.nan_to_num_ +torch.Tensor.ne +torch.Tensor.ne_ +torch.Tensor.not_equal +torch.Tensor.not_equal_ +torch.Tensor.neg +torch.Tensor.neg_ +torch.Tensor.negative +torch.Tensor.negative_ +torch.Tensor.nextafter +torch.Tensor.nextafter_ +torch.Tensor.nonzero +torch.Tensor.norm +torch.Tensor.normal_ +torch.Tensor.orgqr +torch.Tensor.ormqr +torch.Tensor.outer +torch.Tensor.permute +torch.Tensor.pin_memory +torch.Tensor.pinverse +torch.Tensor.polygamma +torch.Tensor.polygamma_ +torch.Tensor.positive +torch.Tensor.pow +torch.Tensor.pow_ +torch.Tensor.prod +torch.Tensor.put_ +torch.Tensor.qr +torch.Tensor.qscheme +torch.Tensor.quantile +torch.Tensor.nanquantile +torch.Tensor.q_scale +torch.Tensor.q_zero_point +torch.Tensor.q_per_channel_scales +torch.Tensor.q_per_channel_zero_points +torch.Tensor.q_per_channel_axis +torch.Tensor.rad2deg +torch.Tensor.random_ +torch.Tensor.ravel +torch.Tensor.reciprocal +torch.Tensor.reciprocal_ +torch.Tensor.record_stream +torch.Tensor.remainder +torch.Tensor.remainder_ +torch.Tensor.renorm +torch.Tensor.renorm_ +torch.Tensor.repeat +torch.Tensor.repeat_interleave +torch.Tensor.reshape +torch.Tensor.reshape_as +torch.Tensor.resize_ +torch.Tensor.resize_as_ +torch.Tensor.roll +torch.Tensor.rot90 +torch.Tensor.round +torch.Tensor.round_ +torch.Tensor.rsqrt +torch.Tensor.rsqrt_ +torch.Tensor.scatter +torch.Tensor.scatter_ +torch.Tensor.scatter_add_ +torch.Tensor.scatter_add +torch.Tensor.scatter_reduce +torch.Tensor.select +torch.Tensor.select_scatter +torch.Tensor.set_ +torch.Tensor.short +torch.Tensor.sigmoid +torch.Tensor.sigmoid_ +torch.Tensor.sign +torch.Tensor.sign_ +torch.Tensor.signbit +torch.Tensor.sgn +torch.Tensor.sgn_ +torch.Tensor.sin +torch.Tensor.sin_ +torch.Tensor.sinc +torch.Tensor.sinc_ +torch.Tensor.sinh +torch.Tensor.sinh_ +torch.Tensor.asinh +torch.Tensor.asinh_ +torch.Tensor.arcsinh +torch.Tensor.arcsinh_ +torch.Tensor.slogdet +torch.Tensor.slice_scatter +torch.Tensor.solve +torch.Tensor.sort +torch.Tensor.split +torch.Tensor.sparse_mask +torch.Tensor.sparse_dim +torch.Tensor.sqrt +torch.Tensor.sqrt_ +torch.Tensor.square +torch.Tensor.square_ +torch.Tensor.squeeze +torch.Tensor.std +torch.std +torch.Tensor.stft +torch.Tensor.sub +torch.Tensor.sub_ +torch.Tensor.subtract +torch.Tensor.subtract_ +torch.Tensor.sum +torch.Tensor.sum_to_size +torch.Tensor.svd +torch.Tensor.swapaxes +torch.Tensor.swapdims +torch.Tensor.symeig +torch.Tensor.t +torch.Tensor.t_ +torch.Tensor.tensor_split +torch.Tensor.tile +torch.Tensor.to +torch.to +torch.Tensor.to_mkldnn +torch.Tensor.take +torch.Tensor.take_along_dim +torch.Tensor.tan +torch.Tensor.tan_ +torch.Tensor.tanh +torch.Tensor.tanh_ +torch.Tensor.atanh +torch.Tensor.atanh_ +torch.Tensor.arctanh +torch.Tensor.arctanh_ +torch.Tensor.topk +torch.Tensor.to_sparse +torch.Tensor.trace +torch.Tensor.transpose +torch.Tensor.transpose_ +torch.Tensor.triangular_solve +torch.Tensor.tril +torch.Tensor.tril_ +torch.Tensor.triu +torch.Tensor.triu_ +torch.Tensor.true_divide +torch.Tensor.true_divide_ +torch.Tensor.trunc +torch.Tensor.trunc_ +torch.Tensor.type +torch.Tensor.type_as +torch.Tensor.unbind +torch.Tensor.unfold +torch.Tensor.uniform_ +torch.Tensor.unique +torch.Tensor.unique_consecutive +torch.Tensor.unsqueeze +torch.Tensor.values +torch.Tensor.var +torch.var +torch.Tensor.vdot +torch.Tensor.view +torch.view +torch.Tensor.view_as +torch.Tensor.vsplit +torch.Tensor.where +torch.Tensor.xlogy +torch.Tensor.xlogy_ +torch.Tensor.zero_ +torch.chunk +torch.distributions.distribution.Distribution.cdf +torch.distributions.distribution.Distribution.entropy +torch.distributions.distribution.Distribution.enumerate_support +torch.distributions.distribution.Distribution.expand +torch.distributions.distribution.Distribution.icdf +torch.distributions.distribution.Distribution.log_prob +torch.distributions.distribution.Distribution.perplexity +torch.distributions.distribution.Distribution.rsample +torch.distributions.distribution.Distribution.sample +torch.distributions.distribution.Distribution.sample_n +torch.distributions.distribution.Distribution.set_default_validate_args +torch.distributions.exp_family.ExponentialFamily.entropy +torch.distributions.bernoulli.Bernoulli.entropy +torch.distributions.bernoulli.Bernoulli.enumerate_support +torch.distributions.bernoulli.Bernoulli.expand +torch.distributions.bernoulli.Bernoulli.log_prob +torch.distributions.bernoulli.Bernoulli.sample +torch.distributions.beta.Beta.entropy +torch.distributions.beta.Beta.expand +torch.distributions.beta.Beta.log_prob +torch.distributions.beta.Beta.rsample +torch.distributions.binomial.Binomial.entropy +torch.distributions.binomial.Binomial.enumerate_support +torch.distributions.binomial.Binomial.expand +torch.distributions.binomial.Binomial.log_prob +torch.distributions.binomial.Binomial.sample +torch.distributions.categorical.Categorical.entropy +torch.distributions.categorical.Categorical.enumerate_support +torch.distributions.categorical.Categorical.expand +torch.distributions.categorical.Categorical.log_prob +torch.distributions.categorical.Categorical.sample +torch.distributions.cauchy.Cauchy.cdf +torch.distributions.cauchy.Cauchy.entropy +torch.distributions.cauchy.Cauchy.expand +torch.distributions.cauchy.Cauchy.icdf +torch.distributions.cauchy.Cauchy.log_prob +torch.distributions.cauchy.Cauchy.rsample +torch.distributions.chi2.Chi2 +torch.distributions.chi2.Chi2.expand +torch.distributions.continuous_bernoulli.ContinuousBernoulli +torch.distributions.continuous_bernoulli.ContinuousBernoulli.cdf +torch.distributions.continuous_bernoulli.ContinuousBernoulli.entropy +torch.distributions.continuous_bernoulli.ContinuousBernoulli.expand +torch.distributions.continuous_bernoulli.ContinuousBernoulli.icdf +torch.distributions.continuous_bernoulli.ContinuousBernoulli.log_prob +torch.distributions.continuous_bernoulli.ContinuousBernoulli.rsample +torch.distributions.continuous_bernoulli.ContinuousBernoulli.sample +torch.distributions.dirichlet.Dirichlet.entropy +torch.distributions.dirichlet.Dirichlet.expand +torch.distributions.dirichlet.Dirichlet.log_prob +torch.distributions.dirichlet.Dirichlet.rsample +torch.distributions.exponential.Exponential.cdf +torch.distributions.exponential.Exponential.entropy +torch.distributions.exponential.Exponential.expand +torch.distributions.exponential.Exponential.icdf +torch.distributions.exponential.Exponential.log_prob +torch.distributions.exponential.Exponential.rsample +torch.distributions.fishersnedecor.FisherSnedecor.expand +torch.distributions.fishersnedecor.FisherSnedecor.log_prob +torch.distributions.fishersnedecor.FisherSnedecor.rsample +torch.distributions.gamma.Gamma.entropy +torch.distributions.gamma.Gamma.expand +torch.distributions.gamma.Gamma.log_prob +torch.distributions.gamma.Gamma.rsample +torch.distributions.geometric.Geometric.entropy +torch.distributions.geometric.Geometric.expand +torch.distributions.geometric.Geometric.log_prob +torch.distributions.geometric.Geometric.sample +torch.distributions.gumbel.Gumbel.entropy +torch.distributions.gumbel.Gumbel.expand +torch.distributions.gumbel.Gumbel.log_prob +torch.distributions.half_cauchy.HalfCauchy +torch.distributions.half_cauchy.HalfCauchy.cdf +torch.distributions.half_cauchy.HalfCauchy.entropy +torch.distributions.half_cauchy.HalfCauchy.expand +torch.distributions.half_cauchy.HalfCauchy.icdf +torch.distributions.half_cauchy.HalfCauchy.log_prob +torch.distributions.half_normal.HalfNormal +torch.distributions.half_normal.HalfNormal.cdf +torch.distributions.half_normal.HalfNormal.entropy +torch.distributions.half_normal.HalfNormal.expand +torch.distributions.half_normal.HalfNormal.icdf +torch.distributions.half_normal.HalfNormal.log_prob +torch.distributions.independent.Independent +torch.distributions.independent.Independent.entropy +torch.distributions.independent.Independent.enumerate_support +torch.distributions.independent.Independent.expand +torch.distributions.independent.Independent.log_prob +torch.distributions.independent.Independent.rsample +torch.distributions.independent.Independent.sample +torch.distributions.kumaraswamy.Kumaraswamy +torch.distributions.kumaraswamy.Kumaraswamy.entropy +torch.distributions.kumaraswamy.Kumaraswamy.expand +torch.distributions.lkj_cholesky.LKJCholesky +torch.distributions.lkj_cholesky.LKJCholesky.expand +torch.distributions.lkj_cholesky.LKJCholesky.log_prob +torch.distributions.lkj_cholesky.LKJCholesky.sample +torch.distributions.laplace.Laplace +torch.distributions.laplace.Laplace.cdf +torch.distributions.laplace.Laplace.entropy +torch.distributions.laplace.Laplace.expand +torch.distributions.laplace.Laplace.icdf +torch.distributions.laplace.Laplace.log_prob +torch.distributions.laplace.Laplace.rsample +torch.distributions.log_normal.LogNormal +torch.distributions.log_normal.LogNormal.entropy +torch.distributions.log_normal.LogNormal.expand +torch.distributions.lowrank_multivariate_normal.LowRankMultivariateNormal +torch.distributions.lowrank_multivariate_normal.LowRankMultivariateNormal.entropy +torch.distributions.lowrank_multivariate_normal.LowRankMultivariateNormal.expand +torch.distributions.lowrank_multivariate_normal.LowRankMultivariateNormal.log_prob +torch.distributions.lowrank_multivariate_normal.LowRankMultivariateNormal.rsample +torch.distributions.mixture_same_family.MixtureSameFamily +torch.distributions.mixture_same_family.MixtureSameFamily.cdf +torch.distributions.mixture_same_family.MixtureSameFamily.expand +torch.distributions.mixture_same_family.MixtureSameFamily.log_prob +torch.distributions.mixture_same_family.MixtureSameFamily.sample +torch.distributions.multinomial.Multinomial +torch.distributions.multinomial.Multinomial.entropy +torch.distributions.multinomial.Multinomial.expand +torch.distributions.multinomial.Multinomial.log_prob +torch.distributions.multinomial.Multinomial.sample +torch.distributions.multivariate_normal.MultivariateNormal +torch.distributions.multivariate_normal.MultivariateNormal.entropy +torch.distributions.multivariate_normal.MultivariateNormal.expand +torch.distributions.multivariate_normal.MultivariateNormal.log_prob +torch.distributions.multivariate_normal.MultivariateNormal.rsample +torch.distributions.negative_binomial.NegativeBinomial +torch.distributions.negative_binomial.NegativeBinomial.expand +torch.distributions.negative_binomial.NegativeBinomial.log_prob +torch.distributions.negative_binomial.NegativeBinomial.sample +torch.distributions.normal.Normal +torch.distributions.normal.Normal.cdf +torch.distributions.normal.Normal.entropy +torch.distributions.normal.Normal.expand +torch.distributions.normal.Normal.icdf +torch.distributions.normal.Normal.log_prob +torch.distributions.normal.Normal.rsample +torch.distributions.normal.Normal.sample +torch.distributions.one_hot_categorical.OneHotCategorical +torch.distributions.one_hot_categorical.OneHotCategorical.entropy +torch.distributions.one_hot_categorical.OneHotCategorical.enumerate_support +torch.distributions.one_hot_categorical.OneHotCategorical.expand +torch.distributions.one_hot_categorical.OneHotCategorical.log_prob +torch.distributions.one_hot_categorical.OneHotCategorical.sample +torch.distributions.pareto.Pareto +torch.distributions.pareto.Pareto.entropy +torch.distributions.pareto.Pareto.expand +torch.distributions.poisson.Poisson +torch.distributions.poisson.Poisson.expand +torch.distributions.poisson.Poisson.log_prob +torch.distributions.poisson.Poisson.sample +torch.distributions.relaxed_bernoulli.RelaxedBernoulli +torch.distributions.relaxed_bernoulli.RelaxedBernoulli.expand +torch.distributions.relaxed_bernoulli.LogitRelaxedBernoulli +torch.distributions.relaxed_bernoulli.LogitRelaxedBernoulli.expand +torch.distributions.relaxed_bernoulli.LogitRelaxedBernoulli.log_prob +torch.distributions.relaxed_bernoulli.LogitRelaxedBernoulli.rsample +torch.distributions.relaxed_categorical.RelaxedOneHotCategorical +torch.distributions.relaxed_categorical.RelaxedOneHotCategorical.expand +torch.distributions.studentT.StudentT +torch.distributions.studentT.StudentT.entropy +torch.distributions.studentT.StudentT.expand +torch.distributions.studentT.StudentT.log_prob +torch.distributions.studentT.StudentT.rsample +torch.distributions.transformed_distribution.TransformedDistribution +torch.distributions.transformed_distribution.TransformedDistribution.cdf +torch.distributions.transformed_distribution.TransformedDistribution.expand +torch.distributions.transformed_distribution.TransformedDistribution.icdf +torch.distributions.transformed_distribution.TransformedDistribution.log_prob +torch.distributions.transformed_distribution.TransformedDistribution.rsample +torch.distributions.transformed_distribution.TransformedDistribution.sample +torch.distributions.uniform.Uniform +torch.distributions.uniform.Uniform.cdf +torch.distributions.uniform.Uniform.entropy +torch.distributions.uniform.Uniform.expand +torch.distributions.uniform.Uniform.icdf +torch.distributions.uniform.Uniform.log_prob +torch.distributions.uniform.Uniform.rsample +torch.distributions.von_mises.VonMises +torch.distributions.von_mises.VonMises.expand +torch.distributions.von_mises.VonMises.log_prob +torch.distributions.von_mises.VonMises.sample +torch.distributions.weibull.Weibull +torch.distributions.weibull.Weibull.entropy +torch.distributions.weibull.Weibull.expand +torch.distributions.wishart.Wishart +torch.distributions.wishart.Wishart.entropy +torch.distributions.wishart.Wishart.expand +torch.distributions.wishart.Wishart.log_prob +torch.distributions.wishart.Wishart.rsample +torch.distributions.kl.kl_divergence +torch.distributions.kl.register_kl +torch.distributions.transforms.AbsTransform +torch.distributions.transforms.AffineTransform +torch.distributions.transforms.ComposeTransform +torch.distributions.transforms.CorrCholeskyTransform +torch.distributions.transforms.ExpTransform +torch.distributions.transforms.IndependentTransform +torch.distributions.transforms.LowerCholeskyTransform +torch.distributions.transforms.PowerTransform +torch.distributions.transforms.ReshapeTransform +torch.distributions.transforms.SigmoidTransform +torch.distributions.transforms.TanhTransform +torch.distributions.transforms.SoftmaxTransform +torch.distributions.transforms.StackTransform +torch.distributions.transforms.StickBreakingTransform +torch.distributions.transforms.Transform +torch.distributions.transforms.Transform.log_abs_det_jacobian +torch.distributions.transforms.Transform.forward_shape +torch.distributions.transforms.Transform.inverse_shape +torch.distributions.constraints.Constraint +torch.distributions.constraints.Constraint.check +torch.distributions.constraints.cat +torch.distributions.constraints.dependent_property +torch.distributions.constraints.greater_than +torch.distributions.constraints.greater_than_eq +torch.distributions.constraints.independent +torch.distributions.constraints.integer_interval +torch.distributions.constraints.interval +torch.distributions.constraints.half_open_interval +torch.distributions.constraints.less_than +torch.distributions.constraints.multinomial +torch.distributions.constraints.stack +torch.distributions.constraint_registry.ConstraintRegistry +torch.distributions.constraint_registry.ConstraintRegistry.register +torch.fft.fft +torch.fft.ifft +torch.fft.fft2 +torch.fft.ifft2 +torch.fft.fftn +torch.fft.ifftn +torch.fft.rfft +torch.fft.irfft +torch.fft.rfft2 +torch.fft.irfft2 +torch.fft.rfftn +torch.fft.irfftn +torch.fft.hfft +torch.fft.ihfft +torch.fft.hfft2 +torch.fft.ihfft2 +torch.fft.hfftn +torch.fft.ihfftn +torch.fft.fftfreq +torch.fft.rfftfreq +torch.fft.fftshift +torch.fft.ifftshift +torch.linalg.norm +torch.linalg.vector_norm +torch.linalg.matrix_norm +torch.linalg.diagonal +torch.linalg.det +torch.linalg.slogdet +torch.linalg.cond +torch.linalg.matrix_rank +torch.linalg.cholesky +torch.linalg.qr +torch.linalg.lu_factor +torch.linalg.eig +torch.linalg.eigvals +torch.linalg.eigh +torch.linalg.eigvalsh +torch.linalg.svd +torch.linalg.svdvals +torch.linalg.solve +torch.linalg.solve_triangular +torch.linalg.lstsq +torch.linalg.inv +torch.linalg.pinv +torch.linalg.matrix_exp +torch.linalg.matrix_power +torch.linalg.cross +torch.linalg.matmul +torch.linalg.multi_dot +torch.linalg.householder_product +torch.linalg.tensorinv +torch.linalg.tensorsolve +torch.linalg.cholesky_ex +torch.linalg.inv_ex +torch.linalg.lu_factor_ex +torch.special.entr +torch.special.erf +torch.special.erfc +torch.special.erfcx +torch.special.erfinv +torch.special.expit +torch.special.expm1 +torch.special.exp2 +torch.special.gammaln +torch.special.gammainc +torch.special.gammaincc +torch.special.polygamma +torch.special.digamma +torch.special.psi +torch.special.i0 +torch.special.i0e +torch.special.i1 +torch.special.i1e +torch.special.logit +torch.special.logsumexp +torch.special.log1p +torch.special.log_softmax +torch.special.multigammaln +torch.special.ndtr +torch.special.ndtri +torch.special.round +torch.special.sinc +torch.special.softmax +torch.special.xlog1py +torch.special.xlogy +torch.special.zeta +torch.optim.Optimizer +torch.optim.Adadelta +torch.optim.Adagrad +torch.optim.Adam +torch.optim.AdamW +torch.optim.AdamW.add_param_group +torch.optim.AdamW.load_state_dict +torch.optim.AdamW.state_dict +torch.optim.AdamW.step +torch.optim.AdamW.zero_grad +torch.optim.SparseAdam +torch.optim.SparseAdam.add_param_group +torch.optim.SparseAdam.load_state_dict +torch.optim.SparseAdam.state_dict +torch.optim.SparseAdam.step +torch.optim.SparseAdam.zero_grad +torch.optim.Adamax +torch.optim.Adamax.add_param_group +torch.optim.Adamax.load_state_dict +torch.optim.Adamax.state_dict +torch.optim.Adamax.step +torch.optim.Adamax.zero_grad +torch.optim.ASGD +torch.optim.ASGD.add_param_group +torch.optim.ASGD.load_state_dict +torch.optim.ASGD.state_dict +torch.optim.ASGD.step +torch.optim.ASGD.zero_grad +torch.optim.LBFGS +torch.optim.LBFGS.add_param_group +torch.optim.LBFGS.load_state_dict +torch.optim.LBFGS.state_dict +torch.optim.LBFGS.step +torch.optim.LBFGS.zero_grad +torch.optim.NAdam +torch.optim.NAdam.add_param_group +torch.optim.NAdam.load_state_dict +torch.optim.NAdam.state_dict +torch.optim.NAdam.step +torch.optim.NAdam.zero_grad +torch.optim.RAdam +torch.optim.RAdam.add_param_group +torch.optim.RAdam.load_state_dict +torch.optim.RAdam.state_dict +torch.optim.RAdam.step +torch.optim.RAdam.zero_grad +torch.optim.RMSprop +torch.optim.RMSprop.add_param_group +torch.optim.RMSprop.load_state_dict +torch.optim.RMSprop.state_dict +torch.optim.RMSprop.step +torch.optim.RMSprop.zero_grad +torch.optim.Rprop +torch.optim.Rprop.add_param_group +torch.optim.Rprop.load_state_dict +torch.optim.Rprop.state_dict +torch.optim.Rprop.step +torch.optim.Rprop.zero_grad +torch.optim.SGD +torch.optim.SGD.add_param_group +torch.optim.SGD.load_state_dict +torch.optim.SGD.state_dict +torch.optim.SGD.step +torch.optim.SGD.zero_grad +torch.optim.lr_scheduler.LambdaLR.get_last_lr +torch.optim.lr_scheduler.LambdaLR.load_state_dict +torch.optim.lr_scheduler.LambdaLR.print_lr +torch.optim.lr_scheduler.LambdaLR.state_dict +torch.optim.lr_scheduler.MultiplicativeLR +torch.optim.lr_scheduler.MultiplicativeLR.get_last_lr +torch.optim.lr_scheduler.MultiplicativeLR.load_state_dict +torch.optim.lr_scheduler.MultiplicativeLR.print_lr +torch.optim.lr_scheduler.MultiplicativeLR.state_dict +torch.optim.lr_scheduler.StepLR +torch.optim.lr_scheduler.StepLR.get_last_lr +torch.optim.lr_scheduler.StepLR.load_state_dict +torch.optim.lr_scheduler.StepLR.print_lr +torch.optim.lr_scheduler.StepLR.state_dict +torch.optim.lr_scheduler.MultiStepLR +torch.optim.lr_scheduler.MultiStepLR.get_last_lr +torch.optim.lr_scheduler.MultiStepLR.load_state_dict +torch.optim.lr_scheduler.MultiStepLR.print_lr +torch.optim.lr_scheduler.MultiStepLR.state_dict +torch.optim.lr_scheduler.ConstantLR +torch.optim.lr_scheduler.ConstantLR.get_last_lr +torch.optim.lr_scheduler.ConstantLR.load_state_dict +torch.optim.lr_scheduler.ConstantLR.print_lr +torch.optim.lr_scheduler.ConstantLR.state_dict +torch.optim.lr_scheduler.LinearLR.get_last_lr +torch.optim.lr_scheduler.LinearLR.load_state_dict +torch.optim.lr_scheduler.LinearLR.print_lr +torch.optim.lr_scheduler.LinearLR.state_dict +torch.optim.lr_scheduler.ExponentialLR.get_last_lr +torch.optim.lr_scheduler.ExponentialLR.load_state_dict +torch.optim.lr_scheduler.ExponentialLR.print_lr +torch.optim.lr_scheduler.ExponentialLR.state_dict +torch.optim.lr_scheduler.CosineAnnealingLR.get_last_lr +torch.optim.lr_scheduler.CosineAnnealingLR.load_state_dict +torch.optim.lr_scheduler.CosineAnnealingLR.print_lr +torch.optim.lr_scheduler.CosineAnnealingLR.state_dict +torch.optim.lr_scheduler.ChainedScheduler.get_last_lr +torch.optim.lr_scheduler.ChainedScheduler.load_state_dict +torch.optim.lr_scheduler.ChainedScheduler.print_lr +torch.optim.lr_scheduler.ChainedScheduler.state_dict +torch.optim.lr_scheduler.SequentialLR.get_last_lr +torch.optim.lr_scheduler.SequentialLR.load_state_dict +torch.optim.lr_scheduler.SequentialLR.print_lr +torch.optim.lr_scheduler.SequentialLR.state_dict +torch.optim.lr_scheduler.CyclicLR.get_last_lr +torch.optim.lr_scheduler.CyclicLR.get_lr +torch.optim.lr_scheduler.CyclicLR.load_state_dict +torch.optim.lr_scheduler.CyclicLR.print_lr +torch.optim.lr_scheduler.CyclicLR.state_dict +torch.optim.lr_scheduler.OneCycleLR.get_last_lr +torch.optim.lr_scheduler.OneCycleLR.load_state_dict +torch.optim.lr_scheduler.OneCycleLR.print_lr +torch.optim.lr_scheduler.OneCycleLR.state_dict +torch.optim.lr_scheduler.CosineAnnealingWarmRestarts +torch.optim.lr_scheduler.CosineAnnealingWarmRestarts.get_last_lr +torch.optim.lr_scheduler.CosineAnnealingWarmRestarts.load_state_dict +torch.optim.lr_scheduler.CosineAnnealingWarmRestarts.print_lr +torch.optim.lr_scheduler.CosineAnnealingWarmRestarts.state_dict +torch.optim.lr_scheduler.CosineAnnealingWarmRestarts.step +torch.quantization.quantize +torch.quantization.quantize_dynamic +torch.quantization.quantize_qat +torch.quantization.prepare +torch.quantization.prepare_qat +torch.quantization.convert +torch.quantization.fuse_modules +torch.quantization.QuantWrapper +torch.quantization.add_quant_dequant +torch.quantization.add_observer_ +torch.quantization.swap_module +torch.quantization.propagate_qconfig_ +torch.quantization.default_eval_fn +torch.quantization.get_observer_dict +torch.quantization.quantize_fx.prepare_fx +torch.quantization.quantize_fx.prepare_qat_fx +torch.quantization.quantize_fx.convert_fx +torch.quantization.quantize_fx.fuse_fx +torch.quantization.observer.ObserverBase.with_args +torch.quantization.observer.ObserverBase.with_callable_args +torch.quantization.observer.MinMaxObserver.calculate_qparams +torch.quantization.observer.MinMaxObserver.forward +torch.quantization.observer.MinMaxObserver.reset_min_max_vals +torch.quantization.observer.PerChannelMinMaxObserver.reset_min_max_vals +torch.quantization.observer.get_observer_state_dict +torch.quantization.observer.load_observer_state_dict +torch.quantization.observer.default_observer +torch.quantization.observer.default_placeholder_observer +torch.quantization.observer.default_debug_observer +torch.quantization.observer.default_weight_observer +torch.quantization.observer.default_histogram_observer +torch.quantization.observer.default_per_channel_weight_observer +torch.quantization.observer.default_dynamic_quant_observer +torch.quantization.observer.default_float_qparams_observer +torch.quantization.fake_quantize.FakeQuantizeBase.with_args +torch.quantization.fake_quantize.default_fake_quant +torch.quantization.fake_quantize.default_weight_fake_quant +torch.quantization.fake_quantize.default_per_channel_weight_fake_quant +torch.quantization.fake_quantize.default_histogram_fake_quant +torch.quantization.fake_quantize.default_fused_act_fake_quant +torch.quantization.fake_quantize.default_fused_wt_fake_quant +torch.quantization.fake_quantize.default_fused_per_channel_wt_fake_quant +torch.quantization.fake_quantize.disable_fake_quant +torch.quantization.fake_quantize.enable_fake_quant +torch.quantization.fake_quantize.disable_observer +torch.quantization.fake_quantize.enable_observer +torch.nn.intrinsic.qat.update_bn_stats +torch.nn.intrinsic.qat.freeze_bn_stats +torch.nn.qat.Conv2d.from_float +torch.nn.qat.Conv3d.from_float +torch.nn.qat.Linear.from_float +torch.nn.quantized.Conv1d.from_float +torch.nn.quantized.Conv2d.from_float +torch.nn.quantized.Conv3d.from_float +torch.nn.quantized.Embedding.from_float +torch.nn.quantized.EmbeddingBag.from_float +torch.nn.quantized.Linear.from_float +torch.nn.quantized.Linear.from_reference +torch.nn.quantized.functional.avg_pool2d +torch.nn.quantized.functional.avg_pool3d +torch.nn.quantized.functional.adaptive_avg_pool2d +torch.nn.quantized.functional.adaptive_avg_pool3d +torch.nn.quantized.functional.conv1d +torch.nn.quantized.functional.conv2d +torch.nn.quantized.functional.conv3d +torch.nn.quantized.functional.interpolate +torch.nn.quantized.functional.linear +torch.nn.quantized.functional.max_pool1d +torch.nn.quantized.functional.max_pool2d +torch.nn.quantized.functional.celu +torch.nn.quantized.functional.leaky_relu +torch.nn.quantized.functional.hardtanh +torch.nn.quantized.functional.hardswish +torch.nn.quantized.functional.threshold +torch.nn.quantized.functional.elu +torch.nn.quantized.functional.hardsigmoid +torch.nn.quantized.functional.clamp +torch.nn.quantized.functional.upsample +torch.nn.quantized.functional.upsample_bilinear +torch.nn.quantized.functional.upsample_nearest +torch.nn.quantized.dynamic.Linear.from_float +torch.ao.ns._numeric_suite.compare_weights +torch.ao.ns._numeric_suite.get_logger_dict +torch.ao.ns._numeric_suite.Logger.forward +torch.ao.ns._numeric_suite.ShadowLogger.forward +torch.ao.ns._numeric_suite.OutputLogger.forward +torch.ao.ns._numeric_suite.Shadow.forward +torch.ao.ns._numeric_suite.Shadow.add +torch.ao.ns._numeric_suite.Shadow.add_scalar +torch.ao.ns._numeric_suite.Shadow.mul +torch.ao.ns._numeric_suite.Shadow.mul_scalar +torch.ao.ns._numeric_suite.Shadow.cat +torch.ao.ns._numeric_suite.Shadow.add_relu +torch.ao.ns._numeric_suite.prepare_model_with_stubs +torch.ao.ns._numeric_suite.compare_model_stub +torch.ao.ns._numeric_suite.get_matching_activations +torch.ao.ns._numeric_suite.prepare_model_outputs +torch.ao.ns._numeric_suite.compare_model_outputs +torch.ao.ns._numeric_suite_fx.OutputLogger.forward +torch.ao.ns._numeric_suite_fx.NSTracer.is_leaf_module +torch.ao.ns._numeric_suite_fx.extract_weights +torch.ao.ns._numeric_suite_fx.add_loggers +torch.ao.ns._numeric_suite_fx.extract_logger_info +torch.ao.ns._numeric_suite_fx.add_shadow_loggers +torch.ao.ns._numeric_suite_fx.extract_shadow_logger_info +torch.ao.ns._numeric_suite_fx.extend_logger_results_with_comparison +torch.ao.ns.fx.utils.compute_sqnr +torch.ao.ns.fx.utils.compute_normalized_l2_error +torch.ao.ns.fx.utils.compute_cosine_similarity +torch.Tensor.coalesce +torch.Tensor.sparse_resize_ +torch.Tensor.sparse_resize_and_clear_ +torch.Tensor.is_coalesced +torch.Tensor.to_dense +torch.sparse_csr_tensor +torch.sparse.sum +torch.sparse.addmm +torch.sparse.sampled_addmm +torch.sparse.mm +torch.sspaddmm +torch.hspmm +torch.smm +torch.sparse.softmax +torch.sparse.log_softmax +torch.Tensor.__add__ +torch.Tensor.__and__ +torch.Tensor.__bool__ +torch.Tensor.__div__ +torch.Tensor.__eq__ +torch.Tensor.__ge__ +torch.Tensor.__gt__ +torch.Tensor.__iadd__ +torch.Tensor.__iand__ +torch.Tensor.__idiv__ +torch.Tensor.__ifloordiv__ +torch.Tensor.__ilshift__ +torch.Tensor.__imod__ +torch.Tensor.__imul__ +torch.Tensor.__ior__ +torch.Tensor.__irshift__ +torch.Tensor.__isub__ +torch.Tensor.__ixor__ +torch.Tensor.__lshift__ +torch.Tensor.__matmul__ +torch.Tensor.__mod__ +torch.Tensor.__mul__ +torch.Tensor.__nonzero__ +torch.Tensor.__or__ +torch.Tensor.__radd__ +torch.Tensor.__rmul__ +torch.Tensor.__rshift__ +torch.Tensor.__sub__ +torch.Tensor.__truediv__ +torch.Tensor.__xor__ \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_blacklists.txt b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_blacklists.txt new file mode 100644 index 000000000..ddacf840a --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_api_blacklists.txt @@ -0,0 +1,7 @@ +torch.Tensor.set_ +torch.Tensor.unsqueeze +torch.norm +torch.Tensor.view +torch.Tensor.view_as +torch.Tensor.copy_ +torch.Tensor.copy diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_distributed_api.txt b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_distributed_api.txt new file mode 100644 index 000000000..7b09b541a --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/data/torch_distributed_api.txt @@ -0,0 +1,12 @@ +torch.distributed.all_reduce +torch.distributed.reduce +torch.distributed.all_gather +torch.distributed.all_gather_into_tensor +torch.distributed.all_gather_object +torch.distributed.gather +torch.distributed.scatter +torch.distributed.reduce_scatter_tensor +torch.distributed.all_to_all_single +torch.distributed.all_to_all +torch.distributed.broadcast +torch.distributed.reduce_op \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/__init__.py new file mode 100644 index 000000000..c39257b25 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/common.py b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/common.py new file mode 100644 index 000000000..32055c331 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/common.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Tuple, Callable +import torch +from loguru import logger +from cc_debugger.common.utils import Tools +from cc_debugger.core.config import base_settings, config, device_settings +from cc_debugger.core.const import Constant +from cc_debugger.core.enums import CCDMode +from cc_debugger.core.exceptions import UnsupportedTypeException +from cc_debugger.models.models import HandlerParams +from cc_debugger.statistics.api_used import api_used_statistic + + +class CommonField: + FUZZ_TENSOR = "fuzz_tensor" + REQUIRES_GRAD = "requires_grad" + MY_HOLD_PLACE = "my_hold_place" + DEVICE = "device" + DISTRIBUTED_OP = "torch.distributed" + + +def type_check(args: Tuple) -> int: + """ + 类型校验 + + """ + for i, arg in enumerate(args): + logger.debug('arg type: {}', type(arg)) + if torch.is_tensor(arg): + # 不处理meta tensor + if arg.is_meta: + continue + if not Tools.is_float(inputs=arg): + continue + return i + if Tools.is_iterable(v=arg): + return i + # 未找到符合要求的类型入参 + return -1 + + +def check(func_id): + if not config.is_running: + return False + # 如果RECORD_FLAG为True,则需统计模型运行中所有算子 + if base_settings.record_flag: + api_used_statistic.used_api_statistic(func_id=func_id) + if base_settings.single_fuzz: + current_api = config.test_apis[config.current_step % len(config.test_apis)] + if current_api != func_id: + return False + # RECORD_FLAG模式下，插装算子分为两类（所有torch算子和待扰动算子），所以此处需过滤非扰动算子 + if func_id.startswith(Constant.TORCH_FLOAT_PREFIX) and func_id not in config.test_apis: + return False + api_used_statistic.called_statistic(func_id=func_id) + # 存储所有调用函数 + funcs = Tools.get_function_names() + api_used_statistic.used_func_statistic(funcs=funcs) + # 判断算子是不是递归调用 + if Tools.is_stack_full(func_id=func_id): + api_used_statistic.not_stack_statistic(func_id=func_id) + logger.warning('[{}] Stack full. Exit staking.', func_id) + return False + return True + + +def data_pre_deal(func_id, func: Callable, *args, **kwargs): + handler_params = HandlerParams() + handler_params.func_id = func_id + handler_params.args = args + handler_params.kwargs = kwargs + handler_params.origin_func = func + index = type_check(args) + if index == -1: + api_used_statistic.not_input_param_statistic(func_id=func_id) + raise UnsupportedTypeException() + handler_params.index = index + return handler_params + + +def is_to_preheat(func_id: str, device: CCDMode = CCDMode.NPU): + if device != CCDMode.NPU: + return False + if not config.api_preheat.get(func_id): + return False + if config.current_step >= device_settings.get(CCDMode.NPU).preheat_step: + return False + return True diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/debugger.py b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/debugger.py new file mode 100644 index 000000000..a2977013c --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/debugger.py @@ -0,0 +1,112 @@ +import traceback +from functools import wraps +from typing import List + +from loguru import logger + +from cc_debugger.core.config import base_settings, config +from cc_debugger.core.const import Constant +from cc_debugger.core.enums import CCDMode +from cc_debugger.core.exceptions import UnsupportedTypeException +from cc_debugger.decorator.common import check, data_pre_deal, is_to_preheat +from cc_debugger.decorator.dec_backward import CustomHookModule +from cc_debugger.decorator.dec_forward import OtherForward, NpuForward +from cc_debugger.models.models import HandlerParams +from cc_debugger.statistics.api_used import api_used_statistic + + +class DebuggerForwardFactory: + forward_debugger = { + CCDMode.CPU: OtherForward, + CCDMode.GPU: OtherForward, + CCDMode.NPU: NpuForward + } + + @staticmethod + def create_debugger(cc_mode: CCDMode = None): + if not cc_mode: + cc_mode = CCDMode.NPU + debugger = DebuggerForwardFactory.forward_debugger.get(cc_mode) + if debugger: + return debugger + else: + raise ValueError(f"Invalid run debugger mode: {base_settings.mode}:{cc_mode}") + + +def start_cc_debugger(handler_params: HandlerParams): + step_start = 0 + if base_settings.range_fuzz: + step_start = base_settings.step_start + result = {} + for device, step in base_settings.ccd_mode.items(): + if (config.current_step - step_start) % (step + 1) == 0: + handler_params.device = device + cc_debugger = DebuggerForwardFactory.create_debugger(device)(handler_params) + result[device] = cc_debugger.torch_api_fuzz() + for device in Constant.PRIORITY: + if device in result.keys(): + return result[device] + raise UnsupportedTypeException() + + +def decorate_forward_function(func, func_id=None): + """ + 装饰器 + """ + + if not func_id: + func_id = func.__name__ + + @wraps(func) + def fuzz_wrapper(*args, **kwargs): + try: + if check(func_id): + handler_params = data_pre_deal(func_id, func, *args, **kwargs) + return start_cc_debugger(handler_params) + except Exception as e: + logger.error('[{}] Error: {}', func_id, e) + logger.error('[{}] Error detail: {}', func_id, traceback.format_exc()) + finally: + api_used_statistic.save_to_file() + return func(*args, **kwargs) + + return fuzz_wrapper + + +def backward_pre_handle(func_id): + step_start = 0 + is_preheat = False + handle_device: List = list() + if base_settings.range_fuzz: + step_start = base_settings.step_start + for device, step in base_settings.ccd_mode.items(): + if (config.current_step - step_start) % (step + 1) != 0: + continue + handle_device.append(device) + is_preheat = is_preheat or is_to_preheat(func_id, device) + return is_preheat, handle_device + + +def decorate_backward_function(func, func_id=None): + if not func_id: + func_id = func.__name__ + + @wraps(func) + def fuzz_wrapper(*args, **kwargs): + try: + logger.info("current step: {}", config.current_step) + is_preheat, handle_device = backward_pre_handle(func_id) + if len(handle_device) > 0 and check(func_id): + handler_params = data_pre_deal(func_id, func, *args, **kwargs) + cus_hook = CustomHookModule(handler_params.func_id, func, is_preheat, handle_device) + result = cus_hook(*args, **kwargs) + api_used_statistic.normal_process_statistic(func_id=func_id) + return result + except Exception as e: + logger.error("{}: {}", func_id, e, exc_info=True) + finally: + api_used_statistic.save_to_file() + return func(*args, **kwargs) + + return fuzz_wrapper + diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_backward.py b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_backward.py new file mode 100644 index 000000000..e4c97dca7 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_backward.py @@ -0,0 +1,263 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Dict +try: + import torch_npu +except ImportError: + pass +import torch +from loguru import logger +from torch import nn +from cc_debugger.core.const import Constant +from cc_debugger.run_api.run_base import RemoteRun +from cc_debugger.decorator.common import CommonField +from cc_debugger.core.enums import HandlerType, CCDMode, InputKey, Apis +from cc_debugger.core.config import base_settings +from cc_debugger.core.exceptions import UnsupportedTypeException +from cc_debugger.common.utils import Tools, convert_tensor_to_device +from cc_debugger.layers.fuzz_layer import FuzzLayer +from cc_debugger.statistics.api_used import api_used_statistic +from cc_debugger.result_handlers.handler_factory import BackwardHandlerFactory + + +class CustomHookModule(nn.Module): + """ + Use the hook pair to add perturbations in reverse and compare the gradients before and after the perturbations + """ + + def __init__(self, api_name, func, is_preheat, handle_device): + super().__init__() + self.api_name = api_name + self.origin_func = func + self.register_backward_hook(self.backward_hook) + self.register_forward_pre_hook(self.forward_hook) + self.backward_input = None + self.args = tuple() + self.kwargs = dict() + self.ccd_grad_input: Dict = dict() + self.origin_grad_input = None + self.is_compare = True + self.need_grad_flag = list() + self.is_preheat = is_preheat + self.handle_device = handle_device + + def get_vjp_input(self): + _input = [] + for v in self.backward_input: + if isinstance(v, dict) and CommonField.FUZZ_TENSOR in v.keys(): + _input.append(torch.tensor(v.get(CommonField.FUZZ_TENSOR).data, + dtype=v.get(CommonField.FUZZ_TENSOR).dtype, + device=v.get(CommonField.DEVICE), + requires_grad=v.get(CommonField.REQUIRES_GRAD))) + else: + _input.append(v) + + _inner_args_tmp = [] + need_grad_tensors = [] + for _i, _v in enumerate(_input): + if torch.is_tensor(_v) and _v.requires_grad: + _inner_args_tmp.append(CommonField.MY_HOLD_PLACE) + need_grad_tensors.append(_v) + self.need_grad_flag.append(True) + logger.debug('my hold place index{} {}', _i, _v.shape) + else: + self.need_grad_flag.append(False) + _inner_args_tmp.append(_v) + _inner_args = tuple(_inner_args_tmp) + return need_grad_tensors, _inner_args + + def get_grad_input_from_vjp(self, need_grad_tensors, grad_output, inner_args, device=None): + """ + 根据vjp计算grad + :param need_grad_tensors: 需要求grad的输入 + :param grad_output: + :param inner_args + :param device + """ + + def my_func(*x): + _real_input = [] + i1 = 0 + dtype = Tools.get_first_tensor(x).dtype + for v in inner_args: + if v == CommonField.MY_HOLD_PLACE: + _real_input.append(x[i1]) + i1 += 1 + elif isinstance(v, torch.Tensor): + _real_input.append(v.to(dtype)) + else: + _real_input.append(v) + kwargs = self.kwargs + if device == CCDMode.CPU: + kwargs = convert_tensor_to_device(self.kwargs) + _real_input = convert_tensor_to_device(_real_input) + return self.origin_func(*_real_input, **kwargs) + + _, grad_input = torch.autograd.functional.vjp(my_func, tuple(need_grad_tensors), grad_output) + return grad_input + + def get_fuzzed_grad_input(self, grad_output, need_grad_tensors, inner_args): + """ + 获取npu扰动后的grad_input + """ + fuzz_layer = FuzzLayer(func_id=self.api_name) + # 得到加了扰动之后的值 + is_added, fuzzed_need_grad_tensors = fuzz_layer(need_grad_tensors) + # 未加扰动的输入直接执行原函数返回 + if not is_added: + api_used_statistic.not_added_statistic(func_id=self.api_name) + raise UnsupportedTypeException() + new_grad_input = self.get_grad_input_from_vjp(tuple(fuzzed_need_grad_tensors), grad_output, inner_args) + self.ccd_grad_input[CCDMode.NPU] = tuple([v.cpu() for v in new_grad_input]) + logger.debug('new grad input num {}', len(self.ccd_grad_input)) + + def get_cpu_grad_input(self, grad_output, need_grad_tensors, inner_args): + """ + 获取cpu或者npu的grad_input + """ + need_grad_tensors = convert_tensor_to_device(need_grad_tensors) + grad_output = convert_tensor_to_device(grad_output) + inner_args = convert_tensor_to_device(inner_args) + self.ccd_grad_input[CCDMode.CPU] = self.get_grad_input_from_vjp(tuple(need_grad_tensors), + grad_output, inner_args) + + def get_gpu_grad_input(self, grad_output): + """ + 获取gpu的grad_input + """ + req_data: Dict = { + InputKey.ARGS.value: convert_tensor_to_device(self.args), + InputKey.GRAD.value: convert_tensor_to_device(grad_output), + InputKey.NEED_GRAD.value: self.need_grad_flag + } + result = RemoteRun.run_remote_result(self.api_name, req_data, Apis.API_AUTOGRAD.value) + self.ccd_grad_input[CCDMode.GPU] = result + + def deal_fuzzed_and_original_result(self, device, original_grad, fuzzed_grad, index, handler_type): + """ + 处理扰动前后算子的结果(check/relative) + :param device: 设备 + :param original_grad: 算子原始梯度 + :param fuzzed_grad: 扰动后梯度 + :param index: 输入的下标 + :param handler_type + """ + cls = BackwardHandlerFactory(which=handler_type).create() + handler = cls(self.api_name, self.ccd_grad_input.get(CCDMode.CPU), device, self.args) + try: + is_consistent = handler.handle(original_grad, fuzzed_grad, index, save_flag=False) + if not is_consistent: + self.is_compare = False + for i, origin_grad in enumerate(self.origin_grad_input): + fuzz_grad = self.ccd_grad_input.get(device)[i].to(origin_grad.device) + result = handler.handle(origin_grad, fuzz_grad, i, save_flag=True) + logger.debug("{} enter compare two vjp, index {} compare_result: {}", self.api_name, i, result) + logger.debug('new grad shape {}', fuzzed_grad.shape) + except Exception as e: + logger.error("[{}] compare failed: {}", self.api_name, e) + + def calculate_grad_input(self, grad_output, need_grad_tensors, inner_args): + """ + 根据base_settings.ccd_mode选择是否获取对应设备的grad_input + """ + for device in self.handle_device: + if device == CCDMode.NPU: + self.get_fuzzed_grad_input(grad_output, need_grad_tensors, inner_args) + continue + if device == CCDMode.CPU: + self.get_cpu_grad_input(grad_output, need_grad_tensors, inner_args) + continue + if device == CCDMode.GPU: + self.get_gpu_grad_input(grad_output) + if self.is_preheat and CCDMode.CPU not in self.handle_device: + self.get_cpu_grad_input(grad_output, need_grad_tensors, inner_args) + + def deal_all_ccd_mode_result(self, original_grad, new_grad_index, index): + """ + 根据base_settings.ccd_mode选择比较对应设备的grad_input + """ + for device in self.handle_device: + handler_type = base_settings.handler_type + if device == CCDMode.NPU and self.is_preheat: + handler_type = HandlerType.PREHEAT + if self.ccd_grad_input.get(device) is None: + logger.debug('Not Found new grads') + continue + try: + with torch.no_grad(): + fuzzed_grad = self.ccd_grad_input.get(device)[new_grad_index].to(original_grad.device) + except IndexError: + logger.error('Index out of range. index: {}, new grads num {}', + new_grad_index, len(self.ccd_grad_input)) + continue + if original_grad.shape != fuzzed_grad.shape: + logger.error('grad shape != new grad shape') + continue + self.deal_fuzzed_and_original_result(device, original_grad, fuzzed_grad, index, handler_type) + + def backward_hook(self, module, grad_input, grad_output): + logger.debug('grad_input len {}', len(grad_input)) + + logger.debug('grad_output len {} shape {}', len(grad_output), grad_output[0].shape, ) + _new_grad_output = grad_output + logger.debug('self.origin func backward hook {}', self.origin_func) + try: + need_grad_tensors, _inner_args = self.get_vjp_input() + # 使用vjp获取原始grad_input + origin_grad_input = self.get_grad_input_from_vjp(tuple(need_grad_tensors), _new_grad_output, _inner_args) + self.origin_grad_input = tuple([v.cpu() for v in origin_grad_input]) + # 使用vjp获取扰动后的grad_input (gpu暂时不用vjp，调用temu的gpu远程服务) + self.calculate_grad_input(_new_grad_output, tuple(need_grad_tensors), _inner_args) + for _i, v in enumerate(grad_input): + if hasattr(v, 'shape'): + logger.debug('grad input i {}, v {}, dtype {}', _i, v.shape, v.dtype) + else: + logger.debug('grad input {}', v) + if base_settings.handler_type != HandlerType.FIX: + return grad_input + for device in Constant.PRIORITY: + if device in self.ccd_grad_input.keys(): + return Tools.convert_fuzz_output_to_origin_output(grad_input, self.ccd_grad_input.get(device)) + except Exception as e: + logger.error('[{}] Error: {}', self.api_name, e) + return grad_input + + def forward(self, *args, **kwargs): + logger.debug('forward origin func {}', self.origin_func) + self.args = args + self.kwargs = kwargs + return self.origin_func(*args, **kwargs) + + def forward_hook(self, module, inputs): + _input = [] + with torch.no_grad(): + for v in inputs: + if torch.is_tensor(v): + _input.append({CommonField.DEVICE: v.device, CommonField.FUZZ_TENSOR: v.cpu(), + CommonField.REQUIRES_GRAD: v.requires_grad}) + else: + _input.append(v) + if base_settings.handler_type == HandlerType.CHECK: + self.compare_model_and_vjp(inputs) + self.backward_input = tuple(_input) + + def compare_model_and_vjp(self, inputs): + _i = 0 + for j, v in enumerate(inputs): + if torch.is_tensor(v) and v.requires_grad: + def fuzz_tensor_grad(grad, new_grad_index=_i, input_index=j): + logger.debug('input index {} requires_index {} grad shape {}', input_index, + new_grad_index, grad.shape) + if not self.is_compare: + # 只要某个输入的grad和扰动的grad不一致，则直接对比两次vjp的结果，其余input的grad不再进行对比 + return grad + try: + self.deal_all_ccd_mode_result(grad, new_grad_index, input_index) + except Exception as e: + logger.error(e) + return grad + return grad + + v.register_hook(fuzz_tensor_grad) + _i += 1 diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_forward.py b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_forward.py new file mode 100644 index 000000000..44e876579 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/decorator/dec_forward.py @@ -0,0 +1,150 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import abc +from abc import abstractmethod + +from loguru import logger +from cc_debugger.common import utils +from cc_debugger.core.config import base_settings, config, device_settings +from cc_debugger.core.enums import PerturbationModeType, CCDMode, InputKey, Apis +from cc_debugger.core.exceptions import UnsupportedTypeException +from cc_debugger.decorator.common import CommonField +from cc_debugger.layers.fuzz_layer import FuzzLayer +from cc_debugger.models.models import HandlerParams, FuncResult +from cc_debugger.result_handlers.handler_factory import ForwardHandlerFactory +from cc_debugger.statistics.api_used import api_used_statistic +from cc_debugger.run_api.run_base import BaseRun, RemoteRun + + +class BaseForward(metaclass=abc.ABCMeta): + + def __init__(self, handler_params: HandlerParams): + self.handler_params = handler_params + + @abstractmethod + def get_result_fuzzed(self, handler_params: HandlerParams): + pass + + def torch_api_fuzz(self): + """ + 装饰器实际执行逻辑 + :return: + """ + args = self.handler_params.args + kwargs = self.handler_params.kwargs + result_fuzzed = self.get_result_fuzzed(self.handler_params) + # 原输入得到原输出 + result_original = self.handler_params.origin_func(*args, **kwargs) + self.handler_params.func_result = FuncResult(result_original=result_original, result_fuzzed=result_fuzzed) + result = self.deal_fuzzed_and_original_result() + logger.info('[{}] Staking completed.', self.handler_params.func_id) + return result + + def get_compare_data(self): + """ + 获取待比较的扰动前后的输出/输入数据 + """ + # 通讯类算子对比输入（添加扰动前后的输入），其余算子对比输出 + if CommonField.DISTRIBUTED_OP not in self.handler_params.func_id: + return + # 提精度的时候，会对所有输入进行扰动，故针对整个args进行比较，其余只针对单个参数 + if device_settings.get(CCDMode.NPU).pert_type == PerturbationModeType.IMPROVE_PRECISION: + fuzzed_value = self.handler_params.fuzzed_value[0] + self.handler_params.func_result.result_original = self.handler_params.args + self.handler_params.func_result.result_fuzzed = fuzzed_value + else: + self.handler_params.func_result = FuncResult(result_original=self.handler_params.args[self. + handler_params.index], + result_fuzzed=self.handler_params.fuzzed_value) + + def deal_fuzzed_and_original_result(self): + """ + 处理扰动前后算子的结果 + """ + result_original = self.handler_params.func_result.result_original + self.get_compare_data() + logger.info("current step: {}", config.current_step) + # 只有需要预热且当前step小于最大预热step且算子需要预热才会进入预热处理（当算子已经满足预热策略条件会提前退出预热，后续将直接走check/fix） + handler = ForwardHandlerFactory(which=base_settings.handler_type).create(func_id=self.handler_params.func_id, + device=self.handler_params.device) + result = handler.handle(params=self.handler_params) + api_used_statistic.normal_process_statistic(func_id=self.handler_params.func_id) + # 由于通讯类算子是比较输入，则handler的返回是输入的结果，此时需要额外处理，直接返回原始输出 + if CommonField.DISTRIBUTED_OP in self.handler_params.func_id: + result = result_original + return result + + +class NpuForward(BaseForward): + @staticmethod + def fuzzed_result(origin_input_output: HandlerParams, fuzzed_value): + """ + 以加了扰动的值作为输入得到对应的输出 + + :param origin_input_output: + :param fuzzed_value: + :return: + """ + args = origin_input_output.args + kwargs = origin_input_output.kwargs + index = origin_input_output.index + func = origin_input_output.origin_func + if device_settings.get(CCDMode.NPU).pert_type == PerturbationModeType.IMPROVE_PRECISION: + result = func(*fuzzed_value[0], **fuzzed_value[1]) + del fuzzed_value + return result + if index == 0: + args2 = args[1:] + if not args2: + result_fuzzed = func(fuzzed_value, **kwargs) + else: + result_fuzzed = func(fuzzed_value, *args2, **kwargs) + else: + args2_front = args[:index] + args2_rear = args[index + 1:] + if not args2_rear: + result_fuzzed = func(*args2_front, fuzzed_value, **kwargs) + else: + result_fuzzed = func(*args2_front, fuzzed_value, *args2_rear, **kwargs) + return result_fuzzed + + def get_result_fuzzed(self, handler_params: HandlerParams): + """ + 对输入添加扰动，获取扰动后的结果 + """ + args = handler_params.args + kwargs = handler_params.kwargs + func_id = handler_params.func_id + fuzz_layer = FuzzLayer(func_id=func_id) + # 得到加了扰动之后的值 + is_added, fuzzed_value = fuzz_layer([args, kwargs] if device_settings.get(CCDMode.NPU).pert_type == + PerturbationModeType.IMPROVE_PRECISION else args[ + handler_params.index]) + # 未加扰动的输入直接执行原函数返回 + if not is_added: + api_used_statistic.not_added_statistic(func_id=func_id) + raise UnsupportedTypeException() + handler_params.fuzzed_value = fuzzed_value + handler_params.input_type = fuzz_layer.input_type + return self.fuzzed_result(origin_input_output=handler_params, fuzzed_value=fuzzed_value) + + +class OtherForward(BaseForward): + def get_result_fuzzed(self, handler_params: HandlerParams): + """ + 将输入输出to到cpu或者gpu上运算获取结果 + """ + args = handler_params.args + kwargs = handler_params.kwargs + func_id = handler_params.func_id + # 得到加了扰动之后的值 + inputs = {InputKey.ARGS.value: args, InputKey.KWARGS.value: kwargs} + inputs = utils.convert_tensor_to_device(inputs) + br = BaseRun(func_id, inputs, handler_params.device.value) + if handler_params.device == CCDMode.CPU: + res = br.cpu_run() + else: + res = RemoteRun.run_remote_result(func_id, inputs, Apis.API_RESULT.value) + handler_params.fuzzed_value = [i for i in inputs.values()] + return res diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/layers/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/layers/__init__.py new file mode 100644 index 000000000..c39257b25 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/layers/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/layers/fuzz_layer.py b/debug/accuracy_tools/CCDebugger/cc_debugger/layers/fuzz_layer.py new file mode 100644 index 000000000..d56592b70 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/layers/fuzz_layer.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Optional, Iterable, Tuple, Any, Callable, Dict +import torch +import torch.nn as nn +from loguru import logger +from torch._C import _VariableFunctions as torchCFunctions +from cc_debugger.core.config import device_settings +from cc_debugger.core.const import ThdConfig, Constant +from cc_debugger.common.custom_iterable import CustomIterable +from cc_debugger.core.enums import TensorFloatType, PerturbationModeType, CCDMode +from cc_debugger.core.exceptions import UnsupportedTypeException +from cc_debugger.common.utils import Tools + + +class FuzzLayer(nn.Module): + """ + 用于添加扰动 + + """ + + def __init__(self, func_id: str, **kwargs): + """ + 保持shape的一致性 + + :param kwargs: + """ + super(FuzzLayer, self).__init__(**kwargs) + # torch api 名称 + self.func_id: str = func_id + # 扰动值 + self.perturbation_value: Optional[float] = None + self.head: int = 0 + self.tail: int = -1 + + # 输入的float类型 + self.input_type: Optional[TensorFloatType] = None + self.is_fuzz: bool = False + + def forward(self, inputs) -> Tuple[bool, Any]: + """ + 添加扰动 + + :param inputs: 原输入 + :return: + """ + func_dict = { + PerturbationModeType.ADD_NOISE: self._add_noise, + PerturbationModeType.CHANGE_VALUE: self._change_value, + PerturbationModeType.IMPROVE_PRECISION: self._improve_precision, + PerturbationModeType.NO_CHANGE: self._no_change + } + logger.debug('[{}] Perturbation mode: {}', self.func_id, device_settings.get(CCDMode.NPU).pert_type) + func: Callable = func_dict.get(device_settings.get(CCDMode.NPU).pert_type) + return func(inputs) + + def _add_noise(self, inputs) -> Tuple[bool, Any]: + """ + 对输入添加噪声 + + :param inputs: + :return: + """ + logger.debug('[{}] Perturbation Mode: add noise.', self.func_id) + if torch.is_tensor(inputs): + # 只对浮点数加扰动 + if not Tools.is_float(inputs): + logger.warning('[{}] Inputs dtype: {}.', self.func_id, inputs.dtype) + return False, inputs + if not Tools.is_add_noise(v=inputs, func_id=self.func_id): + return False, inputs + self._set_perturbation_value(v=inputs) + noise = self._get_noise(inputs) + result = torchCFunctions.where(torchCFunctions.abs(inputs) > self.perturbation_value ** 0.5, + noise + inputs, inputs) + return True, Tools.to_half(v=inputs, result=result, func_id=self.func_id) + elif Tools.is_iterable(inputs): + is_added, inputs = self._deal_with_iterable(inputs) + return is_added, inputs + else: + logger.error('[{}] Unsupported type. type: {}.', self.func_id, type(inputs)) + return False, inputs + + def _get_noise(self, v): + """ + 得到要添加的噪声值 + + """ + dtype = v.dtype + if hasattr(v, Constant.DEVICE): + device = str(v.device) + logger.debug('[{}] Input device: {}.', self.func_id, device) + if device == Constant.META: + noise = torchCFunctions.ones(v.shape, device=Constant.META, dtype=dtype) * self.perturbation_value + else: + noise = torchCFunctions.ones(v.shape, dtype=dtype).to(device) * self.perturbation_value + else: + logger.debug('[{}] Input device: cpu.', self.func_id) + noise = torchCFunctions.ones(v.shape, dtype=dtype) * self.perturbation_value + return noise + + def _add_noise_with_dict_tensor(self, key, value, custom_iterable: CustomIterable) -> bool: + if not Tools.is_float(value): + custom_iterable.add_pair(key, value) + return False + if not Tools.is_add_noise(v=value, func_id=self.func_id): + custom_iterable.add_pair(key, value) + return False + self._set_perturbation_value(v=value) + noise = self._get_noise(value) + result = torchCFunctions.where(torchCFunctions.abs(value) > self.perturbation_value ** 0.5, + noise + value, value) + custom_iterable.add_pair(key, Tools.to_half(v=value, result=result, func_id=self.func_id)) + return True + + def _add_noise_with_list_tensor(self, value, custom_iterable: CustomIterable): + if not Tools.is_float(value): + custom_iterable.append(value) + return False + if not Tools.is_add_noise(v=value, func_id=self.func_id): + custom_iterable.append(value) + return False + self._set_perturbation_value(v=value) + noise = self._get_noise(value) + result = torchCFunctions.where(torchCFunctions.abs(value) > self.perturbation_value ** 0.5, + noise + value, value) + custom_iterable.append(Tools.to_half(v=value, result=result, func_id=self.func_id)) + return True + + def _deal_with_iterable(self, t: Iterable) -> Tuple[bool, Any]: + """ + 处理可迭代对象 + + :param t: + :return: + """ + custom_iterable = CustomIterable(iterable=t, func_id=self.func_id) + if custom_iterable.container is None: + logger.error('[{}] Deal with iterable failed. container is None. iterable type: {}', self.func_id, type(t)) + return False, t + # 已经添加扰动次数 只针对第一个满足要求的tensor添加扰动 + disturbances_count = 0 + is_added = False + for _, v in enumerate(t): + if isinstance(t, Dict): + key = v + value = t[v] + if Tools.is_iterable(value): + is_added, new_value = self._deal_with_iterable(value) + custom_iterable.add_pair(key, new_value) + continue + elif torch.is_tensor(value): + is_added = self._add_noise_with_dict_tensor(key, value, custom_iterable) + disturbances_count += 1 if is_added else 0 + else: + custom_iterable.add_pair(key, value) + elif disturbances_count == 0 and torch.is_tensor(v): + is_added = self._add_noise_with_list_tensor(v, custom_iterable) + disturbances_count += 1 if is_added else 0 + else: + custom_iterable.append(v) + return is_added, custom_iterable.get_container() + + def _set_perturbation_value(self, v: torch.Tensor): + """ + 根据不同类型的浮点数来确定不同位数的扰动值 + bfloat16: 1e-4 float16: 1e-6 float32: 1e-8 float64: 1e-16 + + :param v: + :return: + """ + dtype_str = str(v.dtype) + perturbation_value = ThdConfig.PERTURBATION_VALUE_DICT.get(v.dtype) + if perturbation_value: + self.perturbation_value = perturbation_value + self.input_type = TensorFloatType(dtype_str.replace(Constant.TORCH_FLOAT_PREFIX, '')) + else: + raise UnsupportedTypeException() + + def _improve_tensor_precision(self, tensor_obj): + if isinstance(tensor_obj, torch.Tensor) and torch.is_floating_point(tensor_obj) and \ + tensor_obj.dtype != torch.float64: + self._set_improve_value(tensor_obj) + # 精度变化：非float64精度则向上提升，float64不fuzz(float64->float32?) + tensor_obj = self._change_dtype(tensor_obj) + self.is_fuzz = True + return tensor_obj + if isinstance(tensor_obj, dict): + return {k: self._improve_tensor_precision(v) for k, v in tensor_obj.items()} + if isinstance(tensor_obj, (tuple, list)): + return type(tensor_obj)([self._improve_tensor_precision(v) for v in tensor_obj]) + return tensor_obj + + def _improve_precision(self, inputs) -> Tuple[bool, Any]: + """ + 对输入提升精度 + """ + logger.debug('[{}] Perturbation Mode: improve precision.', self.func_id) + fuzz_value = self._improve_tensor_precision(inputs) + return self.is_fuzz, fuzz_value + + @staticmethod + def _no_change(inputs) -> Tuple[bool, Any]: + """ + :param inputs: 待扰动输入 + :return: 元组【是否被扰动，扰动后输入】 + """ + return True, inputs + + def _change_dtype(self, inputs): + if hasattr(inputs, 'device'): + device = inputs.device + if device == "meta": + fi = inputs.to(device="meta", dtype=self.perturbation_value) + else: + fi = inputs.to(dtype=self.perturbation_value).to(device) + else: + fi = inputs.to(dtype=self.perturbation_value) + return fi + + def _set_improve_value(self, inputs): + if str(inputs.dtype) == "torch.float32": + self.perturbation_value = torch.float64 + if str(inputs.dtype) == "torch.float16": + self.perturbation_value = torch.float32 + if str(inputs.dtype) == "torch.bfloat16": + self.perturbation_value = torch.float32 + + def _change_value(self, inputs) -> Tuple[bool, Any]: + """ + 交换输入的首尾 + + :param inputs: + :return: + """ + logger.info('[{}] Perturbation Mode: change value.', self.func_id) + new_inputs = torch.clone(inputs) + temp_first = torch.clone(new_inputs[self.head]) + temp_last = torch.clone(new_inputs[self.tail]) + if isinstance(inputs, (tuple, list)): + new_inputs_temp = list(new_inputs) + new_inputs_temp[self.head] = temp_last + new_inputs_temp[self.tail] = temp_first + return True, type(inputs)(new_inputs_temp) + elif isinstance(inputs, torch.Tensor): + new_inputs[self.head] = temp_last + new_inputs[self.tail] = temp_first + return True, new_inputs + else: + logger.error('[{}] Unsupported type. type: {}, value: {}', self.func_id, type(inputs), inputs) + return False, inputs diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/main.py b/debug/accuracy_tools/CCDebugger/cc_debugger/main.py new file mode 100644 index 000000000..0c37841b5 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/main.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import importlib +import inspect +from functools import wraps +from pathlib import Path +try: + import torch_npu +except ImportError: + pass +import torch +from loguru import logger +from torch.jit._state import disable as jit_disable +from cc_debugger.statistics.api_preheat_counter import preheat_counter +from cc_debugger.statistics.api_used import api_used_statistic +from cc_debugger.core.config import base_settings, config, device_settings +from cc_debugger.core.enums import CCDMode, ModeType +from cc_debugger.decorator.debugger import decorate_forward_function, decorate_backward_function + +_ROUTER = { + ModeType.FORWARD: decorate_forward_function, + ModeType.BACKWARD: decorate_backward_function +} + + +def on_before_iter(): + config.current_step += 1 + + is_preheat = device_settings.get(CCDMode.NPU).is_preheat + if config.current_step == 0: + # cdd在step==0时才开始正式执行 + config.is_running = True + # 初始化预热相关参数 + if is_preheat and CCDMode.NPU in base_settings.ccd_mode: + config.api_preheat = {i: True for i in config.test_apis} + elif config.current_step == 1 and is_preheat: + # 统计未被预热的api + no_preheat: list = api_used_statistic.func_no_preheat_list_statistic() + if len(no_preheat) > 0: + logger.warning('some apis will not be preheated: {}', ','.join(no_preheat)) + # 每一步都要重置预热计数器 + if is_preheat: + preheat_counter.reset() + # 判断是否是范围插装，如是则只插装指定范围step + if base_settings.range_fuzz: + if config.current_step < base_settings.step_start or config.current_step > base_settings.step_end: + config.is_running = False + + +def iter_tracer(func): + @wraps(func) + def func_wrapper(*args, **kwargs): + result = func(*args, **kwargs) + on_before_iter() + return result + + return func_wrapper + + +def cc_debugger(): + """ + 插桩入口 + + :return: + """ + if not base_settings.ccd_flag: + return + jit_disable() + torch.utils.data.dataloader._BaseDataLoaderIter.__next__ = iter_tracer(torch.utils.data.dataloader. + _BaseDataLoaderIter.__next__) + blacklists = list() + fuzz_apis = list() + + logger.info('Start fuzz plugging.') + data_path = Path(__file__).parent.joinpath("data") + blacklists_file_path = data_path.joinpath('torch_api_blacklists.txt') + with open(blacklists_file_path, 'r') as f: + for line in f: + blacklists.append(line.strip()) + + fuzz_api_file_path = data_path.joinpath('torch_api.txt') + with open(fuzz_api_file_path, 'r') as f: + for line in f: + line = line.strip() + if line not in blacklists: + fuzz_apis.append(line.strip()) + config.test_apis = fuzz_apis + + api_file_path = data_path.joinpath('torch_api_all.txt' if base_settings.record_flag else "torch_api.txt") + with open(api_file_path, 'r') as f: + lines = f.readlines() + for line in lines: + hijack(line.strip()) + + +def get_wrapper_obj(orig_func, api_name, module_obj): + if inspect.isclass(orig_func): + wrapped_obj = orig_func + # for torch.Tensor.xx, it has no __init__ + elif module_obj == torch.Tensor and callable(orig_func): + wrapped_obj = _ROUTER.get(base_settings.mode)(orig_func, api_name) + elif callable(orig_func): + wrapped_obj = _ROUTER.get(base_settings.mode)(orig_func, api_name) + else: + wrapped_obj = orig_func + return wrapped_obj + + +def get_module(api_name): + func_name_list = api_name.split('.') + func_name = func_name_list[-1] + module_obj = importlib.import_module(func_name_list[0]) + for i, module_name in enumerate(func_name_list[1:-1]): + if not hasattr(module_obj, module_name): + importlib.import_module(f"{'.'.join(func_name_list[:i + 2])}") + module_obj = getattr(module_obj, module_name) + orig_func = getattr(module_obj, func_name) + + return module_obj, orig_func + + +def hijack(api_name): + if len(api_name.strip()) == 0: + return + try: + func_name = api_name.split('.')[-1] + module_obj, origin_func = get_module(api_name) + wrapped_obj = get_wrapper_obj(origin_func, api_name, module_obj) + setattr(module_obj, func_name, wrapped_obj) + except (AttributeError, ModuleNotFoundError, AssertionError, TypeError) as e: + logger.warning("Failed decorator {}: {}", api_name, e) + except Exception as e: + logger.error("Failed decorator {}: {}", api_name, e) diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/models/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/models/__init__.py new file mode 100644 index 000000000..c39257b25 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/models/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/models/models.py b/debug/accuracy_tools/CCDebugger/cc_debugger/models/models.py new file mode 100644 index 000000000..74ffb4ec8 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/models/models.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Optional, Any, Tuple, Dict, Callable +from pydantic import BaseModel +from cc_debugger.core.enums import TensorFloatType, CCDMode + + +class NormResult(BaseModel): + """ + + """ + + bfloat16: Optional[float] = None + float16: Optional[float] = None + float32: Optional[float] = None + float64: Optional[float] = None + + def set_value(self, norm: float, intput_type: TensorFloatType): + self.__setattr__(name=intput_type.value, value=norm) + + def get_value(self, intput_type: TensorFloatType) -> float: + return getattr(self, intput_type.value) + + +class TauResult(BaseModel): + """ + tau结果类 + + """ + + one_norm: Optional[NormResult] = None + two_norm: Optional[NormResult] = None + endless_norm: Optional[NormResult] = None + + +class Taus(BaseModel): + """ + 三个范数 + + """ + # 一范数 + tau_one: float = 0.0 + # 二范数 + tau_two: float = 0.0 + # 无穷范数 + tau_endless: float = 0.0 + + +class FuncResult(BaseModel): + """ + 被插桩函数的返回值 + + """ + # 无扰动值得到的输出 （未加扰动的输入）（输入到待测函数后的输出） + result_original: Optional[Any] = None + # 添加扰动得到的输出 （扰动后的输入）（输入到待测函数后的输出） + result_fuzzed: Optional[Any] = None + + +class HandlerParams(BaseModel): + """ + 参数集合体 + + """ + func_id: Optional[str] = None + args: Optional[Tuple] = None + kwargs: Optional[Dict] = None + index: Optional[int] = None + func_result: Optional[FuncResult] = None + input_type: Optional[TensorFloatType] = None + origin_func: Optional[Callable] = None + fuzzed_value: Optional[Any] = None + device: Optional[CCDMode] = None + + +class FuzzResult(BaseModel): + """ + + """ + id: Optional[str] = None + fuzz_mean_error: Optional[Any] = None + fuzz_max_error: Optional[Any] = None + fuzz_max_error_instance: Optional[Any] = None + args: Optional[Any] = None + kwargs: Optional[Any] = None + result_original: Optional[Any] = None + result_fuzzed: Optional[Any] = None + tau_dict: Optional[Any] = None diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/__init__.py new file mode 100644 index 000000000..c39257b25 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/abs_handler.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/abs_handler.py new file mode 100644 index 000000000..1eb167a71 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/abs_handler.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +抽象的结果处理类 + +""" +import abc +import os +import pickle +from copy import deepcopy +from typing import Any +import torch +from loguru import logger +from cc_debugger.core.config import config, base_settings +from cc_debugger.core.const import Constant +from cc_debugger.core.enums import TensorFloatType +from cc_debugger.models.models import HandlerParams, NormResult +from cc_debugger.common.utils import Tools + + +class AbsHandler(metaclass=abc.ABCMeta): + + def __init__(self, func_id): + self.func_id = func_id + + @staticmethod + def generate_norm_result(norm: float, input_type: TensorFloatType) -> NormResult: + """ + 构建NormResult类 + + :param norm: 范数 + :param input_type: cc_debugger.decorate_torch.TensorFloatType类 + :return: cc_debugger.decorate_torch.NormResult类 + """ + norm_result = NormResult() + norm_result.set_value(norm=norm, intput_type=input_type) + return norm_result + + @staticmethod + def param_copy(param_value): + """ + Copy Parameter Values. + :param param_value: param value. + :return: param_value duplicate + """ + if isinstance(param_value, torch.Tensor): + return param_value.detach().to('cpu') + if isinstance(param_value, (list, tuple)): + return type(param_value)(AbsHandler.param_copy(i) for i in param_value) + if isinstance(param_value, dict): + return {k: AbsHandler.param_copy(v) for k, v in param_value.items()} + return deepcopy(param_value) + + @abc.abstractmethod + def handle(self, params: HandlerParams) -> Any: + pass + + @staticmethod + def dump_inputs(func_id, file_path: str, *args, **kwargs): + """ + dump输入数据 + """ + param_dict = dict() + try: + param_dict['args'] = AbsHandler.param_copy(args) + param_dict['kwargs'] = AbsHandler.param_copy(kwargs) + + with os.fdopen(os.open(file_path, Constant.FILE_FLAGS, Constant.FILE_MODES), "wb") as f: + pickle.dump(param_dict, f) + except Exception as e: + logger.error("{} save inputs error: {}", func_id, e) + + @staticmethod + def dump_outputs(func_id, file_path: str, outputs): + try: + with os.fdopen(os.open(file_path, Constant.FILE_FLAGS, Constant.FILE_MODES), "wb") as f: + pickle.dump(outputs, f) + except Exception as e: + logger.error("{} save outputs error: {}", func_id, e) + + def save_unequal_result(self, params: HandlerParams, ratio: list, file_path=config.check_dir): + """ + 和已知的范数进行比较 得到当前的最大值 + :param params: + :param ratio: + :param file_path: 文件输出地址 + """ + device_path = file_path.joinpath(f'{base_settings.mode.value}_{params.device.value}') + dir_name = f'{os.getpid()}_{self.func_id}' + + ratio_str = ','.join([str(r) for r in ratio]) + if len(ratio_str) != 0: + dir_name += f'_{ratio_str}' + + dir_path = device_path.joinpath(dir_name) + os.makedirs(dir_path, exist_ok=True) + + self.dump_inputs(self.func_id, f'{dir_path}/inputs.p', params.args, params.kwargs) + self.dump_outputs(self.func_id, f'{dir_path}/fuzz_outputs.p', params.func_result.result_fuzzed) + self.dump_outputs(self.func_id, f'{dir_path}/origin_outputs.p', params.func_result.result_original) + self.save_unequal_result_summary(device_path, self.func_id, ratio_str) + + logger.debug('[{}] Maximum value has been refreshed.', self.func_id) + + @staticmethod + def save_unequal_result_summary(file_path: str, func_id: str, ratio_str: str = ''): + logger.info('save_unequal_result_summary') + record = (os.getpid(), base_settings.mode.value, func_id, ratio_str) + result_header = ('pid', 'mode', 'func_id', 'ratio') + csv_output_path = f'{file_path}/unequal_result_summary.csv' + Tools.write_data_to_csv([record], result_header, csv_output_path) diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/backward_handler.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/backward_handler.py new file mode 100644 index 000000000..4a51aa325 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/backward_handler.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import os +import torch +from loguru import logger +from cc_debugger.core.config import config, device_settings, base_settings +from cc_debugger.core.const import ThdConfig +from cc_debugger.core.enums import PerturbationModeType, CCDMode +from cc_debugger.core.exceptions import UnsupportedTypeException +from cc_debugger.common.utils import Tools, compare_seq +from cc_debugger.result_handlers.check_handler import CheckHandler +from cc_debugger.result_handlers.preheat_handler import PreheatHandler +from cc_debugger.result_handlers.relative_handler import RelativeHandler +from cc_debugger.statistics.api_preheat_counter import preheat_counter +from cc_debugger.statistics.api_used import api_used_statistic + +torch.set_printoptions(precision=10) + + +class BackwardHandler: + """ + BackwardHandler + + """ + + def __init__(self, func_id, preheat_grad, device, *args): + self.func_id = func_id + self.preheat_grad = preheat_grad + self.device: CCDMode = device + self.thd = 1.0 + self.args = args + + def handle(self, origin_grad, fuzz_grad, index, save_flag=False): + is_consistent = True + try: + # check_handle只针对浮点型 + if isinstance(fuzz_grad, bool) or not Tools.check_output_type(fuzz_grad): + return True + if self.device == CCDMode.NPU: + ratio, is_consistent = CheckHandler.npu_compare(origin_grad, fuzz_grad, self.func_id) + else: + is_consistent = CheckHandler.other_compare(origin_grad, fuzz_grad, self.func_id, self.device) + ratio = [] + if not is_consistent and save_flag: + self.save_unequal_result(origin_grad, fuzz_grad, ratio, index) + except Exception as e: + logger.error(e) + return is_consistent + + def get_compare_ratio(self, origin_grad, fuzz_grad): + if isinstance(origin_grad, bool) or not Tools.check_output_type(origin_grad): + raise UnsupportedTypeException + dtype = Tools.get_first_tensor(origin_grad).dtype + # 获取数据的dtype并得到对应的精度e, 默认精度为0.0001 + if device_settings.get(CCDMode.NPU).pert_type == PerturbationModeType.NO_CHANGE.value: + self.thd = ThdConfig.COMP_CONSISTENT + else: + self.thd = ThdConfig.DTYPE_PRE_THD.get(dtype, ThdConfig.DTYPE_PRE_THD.get(torch.float32)) + new_thd = config.api_thd.get(self.func_id) + self.thd = float(new_thd) if new_thd and float(new_thd) > self.thd else self.thd + ratio = Tools.all_ratio_calculate(origin_grad, fuzz_grad) + logger.debug("[{}] ratios [{}]", self.func_id, ratio) + return ratio + + def save_unequal_result(self, origin_grad, fuzz_grad, ratio, index): + """ + 保存不一致的输入输出 + :param origin_grad: + :param fuzz_grad: + :param ratio: + :param index + """ + # dump fuzz和原始输入和输出数据 + mode_path = config.check_dir.joinpath(f'{base_settings.mode.value}_{self.device.value}') + dir_name = f'{os.getpid()}_{self.func_id}' + ratio_str = ','.join([str(r) for r in ratio]) + file_path = mode_path.joinpath(f"{dir_name}_{ratio_str}") + os.makedirs(file_path, exist_ok=True) + + input_path = f'{file_path}/inputs.p' + CheckHandler.dump_inputs(self.func_id, input_path, self.args, None) + fuzz_output_path = f'{file_path}/fuzz_grad_{index}.p' + CheckHandler.dump_outputs(self.func_id, fuzz_output_path, fuzz_grad) + origin_output_path = f'{file_path}/origin_grad_{index}.p' + CheckHandler.dump_outputs(self.func_id, origin_output_path, origin_grad) + + CheckHandler.save_unequal_result_summary(str(mode_path), self.func_id, str(ratio)) + + logger.debug('[{}] Maximum value has been refreshed.', self.func_id) + + +class BackwardPreheatHandler(BackwardHandler): + def handle(self, origin_grad, fuzz_grad, index, save_flag=False): + """ + 预热只走vjp模式 + """ + if not save_flag: + return False + # 第0步统计各个api被调用的次数，用于预热采样 + if config.current_step == 0: + api_used_statistic.one_step_used_func_dict_statistic(func_id=self.func_id) + return True + # 当前step中func_id被调用的次数+1，用于预热采样 + preheat_counter.add_api_called_time(self.func_id) + is_consistent = True + if not PreheatHandler.is_take_a_sample(self.func_id): + return is_consistent + logger.debug("{} enter backward preheat", self.func_id) + ratio, is_consistent = CheckHandler.npu_compare(origin_grad, fuzz_grad, self.func_id) + npu_and_cpu_cons = compare_seq(origin_grad, self.preheat_grad[index]) + logger.debug("{}: fuzz_ratio: {}, is_cons: {}", self.func_id, ratio, npu_and_cpu_cons) + PreheatHandler.preheat(ratio, npu_and_cpu_cons, self.func_id) + if not is_consistent and save_flag: + self.save_unequal_result(origin_grad, fuzz_grad, ratio, index) + return is_consistent + + +class BackwardRelativeHandler(BackwardHandler): + """ + 直方图只走vjp模式 + """ + def handle(self, origin_grad, fuzz_grad, index, save_flag=False): + if not save_flag: + return False + input_dtype = Tools.get_first_tensor(self.args).dtype + RelativeHandler.err_dist_hist(origin_grad, fuzz_grad, input_dtype, self.func_id) + return True + + diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/check_handler.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/check_handler.py new file mode 100644 index 000000000..f641c4fe2 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/check_handler.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Any, Tuple +import torch +from loguru import logger +from cc_debugger.common import utils +from cc_debugger.core.config import device_settings, config +from cc_debugger.common.utils import Tools +from cc_debugger.core.const import ThdConfig +from cc_debugger.core.enums import PerturbationModeType, CCDMode +from cc_debugger.models.models import HandlerParams +from cc_debugger.result_handlers.abs_handler import AbsHandler + +torch.set_printoptions(precision=10) + + +class CheckHandler(AbsHandler): + """ + CheckLayer的handler + + """ + + @staticmethod + def npu_compare(origin_output, fuzz_output, func_id) -> Tuple[list, bool]: + """ + npu使用比值，其余使用相对误差 + """ + dtype = Tools.get_first_tensor(fuzz_output).dtype + # 获取数据的dtype并得到对应的精度e, 默认精度为0.0001 + if device_settings.get(CCDMode.NPU).pert_type == PerturbationModeType.NO_CHANGE: + e = ThdConfig.COMP_CONSISTENT + else: + e = ThdConfig.DTYPE_PRE_THD.get(dtype, ThdConfig.DTYPE_PRE_THD.get(torch.float32)) + new_thd = config.api_thd.get(func_id) + e = float(new_thd) if new_thd and float(new_thd) > e else e + ratio = Tools.all_ratio_calculate(origin_output, fuzz_output) + logger.debug("[{}] ratio [{}]", func_id, ratio) + is_consistent = True + for r in ratio: + if r > e or r < 1 / e: + is_consistent = False + break + return ratio, is_consistent + + @staticmethod + def other_compare(origin_output, fuzz_output, func_id, device) -> bool: + is_consistent = utils.compare_seq(origin_output, fuzz_output) + logger.debug("[{}] [{}] [{}]", func_id, device, is_consistent) + return is_consistent + + def handle(self, params: HandlerParams) -> Any: + """ + + :param params: cc_debugger.models.models.HandlerParams类 + :return: + """ + # 计算出当前api的2个ratio，y1：y2 + fuzz_output = params.func_result.result_fuzzed + original_output = params.func_result.result_original + try: + # check_handle只针对浮点型 + if isinstance(fuzz_output, bool) or not Tools.check_output_type(fuzz_output): + return params.func_result.result_original + if params.device == CCDMode.NPU: + ratio, is_consistent = self.npu_compare(original_output, fuzz_output, self.func_id) + if not is_consistent: + self.save_unequal_result(params, ratio) + else: + is_consistent = self.other_compare(original_output, fuzz_output, self.func_id, params.device) + if not is_consistent: + self.save_unequal_result(params, []) + except Exception as e: + logger.error(e) + return params.func_result.result_original + logger.debug('[{}] Check layer finished.', self.func_id) + # 和已知的范数进行比较 得到当前的最大值 + return params.func_result.result_original diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/fix_handler.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/fix_handler.py new file mode 100644 index 000000000..851ee8b6f --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/fix_handler.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Any +from loguru import logger +from cc_debugger.common.utils import Tools +from cc_debugger.models.models import HandlerParams +from cc_debugger.result_handlers.check_handler import CheckHandler + + +class FixHandler(CheckHandler): + """ + FixLayer的handler + + """ + + # 这里传入精度参数e，默认万分之一 + # fixhandle不同于checkhandle，当检测到r不属于区间时，会把高精度的结果返回。 + + def handle(self, params: HandlerParams) -> Any: + """ + + :param params: cc_debugger.models.models.HandlerParams类 + :return: + """ + # bai chuan测试中，直接返回高精度结果 + # 获取数据的dtype并得到对应的精度e, 默认精度为0.0001 + origin_output = params.func_result.result_original + fuzz_output = params.func_result.result_fuzzed + try: + logger.debug('[{}] fix handler start.', self.func_id) + return Tools.convert_fuzz_output_to_origin_output(origin_output, fuzz_output) + except Exception as e: + logger.error(e) + return params.func_result.result_original + + diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/handler_factory.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/handler_factory.py new file mode 100644 index 000000000..a14781197 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/handler_factory.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +工厂类 + +""" +from cc_debugger.decorator.common import is_to_preheat +from cc_debugger.result_handlers.backward_handler import BackwardHandler, BackwardRelativeHandler +from cc_debugger.result_handlers.backward_handler import BackwardPreheatHandler +from cc_debugger.result_handlers.preheat_handler import PreheatHandler +from cc_debugger.result_handlers.relative_handler import RelativeHandler +from cc_debugger.core.enums import HandlerType, CCDMode +from cc_debugger.core.exceptions import UnsupportedTypeException +from cc_debugger.result_handlers.check_handler import CheckHandler +from cc_debugger.result_handlers.fix_handler import FixHandler + + +class ForwardHandlerFactory: + """ + Forward Handler的工厂类 + + """ + + def __init__(self, which): + self.which = which + + result_handlers = { + HandlerType.CHECK: CheckHandler, + HandlerType.FIX: FixHandler, + HandlerType.RELATIVE: RelativeHandler, + HandlerType.PREHEAT: PreheatHandler + } + + def create(self, func_id: str, device: CCDMode = CCDMode.NPU): + if is_to_preheat(func_id, device): + return PreheatHandler(func_id=func_id) + handler = self.result_handlers.get(self.which) + if handler: + return handler(func_id=func_id) + else: + raise UnsupportedTypeException() + + +class BackwardHandlerFactory: + """ + backward Handler的工厂类 + + """ + + def __init__(self, which): + self.which = which + + result_handlers = { + HandlerType.CHECK: BackwardHandler, + HandlerType.RELATIVE: BackwardRelativeHandler, + HandlerType.PREHEAT: BackwardPreheatHandler + } + + def create(self): + handler = self.result_handlers.get(self.which) + if handler: + return handler + else: + raise UnsupportedTypeException() diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/preheat_handler.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/preheat_handler.py new file mode 100644 index 000000000..edeaa9690 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/preheat_handler.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import torch +import math +from typing import Any +from loguru import logger +from cc_debugger.core.config import device_settings, config +from cc_debugger.common.utils import Tools, convert_tensor_to_device, compare_seq +from cc_debugger.core.const import ThdConfig +from cc_debugger.core.enums import CCDMode +from cc_debugger.models.models import HandlerParams +from cc_debugger.result_handlers.check_handler import CheckHandler +from cc_debugger.statistics.api_used import api_used_statistic +from cc_debugger.statistics.api_preheat_counter import preheat_counter + +torch.set_printoptions(precision=10) + + +class PreheatHandler(CheckHandler): + """ + CheckLayer的handler + + """ + + @staticmethod + def adjust_threshold(step, func_id): + """ + 更新阈值 + 如果当前step已经检测出反例，则退出预热模式 + """ + # 当前step是否进行衰减 + con_ratio = list() + incon_ratio = list() + for ratio, is_correct in config.api_compare.get(step).get(func_id): + if is_correct: + con_ratio.append(ratio) + else: + incon_ratio.append(ratio) + # 只有正例和负例都存在 + if con_ratio and incon_ratio: + if min(incon_ratio) > max(con_ratio): + config.api_thd[func_id] = min(min(incon_ratio), config.api_thd[func_id]) + config.api_preheat[func_id] = False + else: + # TODO 正负例阈值交叉 + pass + # 只存在正例 + elif con_ratio: + # 存在误报 + if max(con_ratio) > config.api_thd[func_id]: + # TODO 对cpu结果升精度，检验是否cpu误报 + # TODO cpu误报，打印cpu异常，忽视该算子 + config.api_thd[func_id] = 1 + ((config.api_thd[func_id] - 1) * ThdConfig.API_THD_STEP) + else: + config.api_thd[func_id] = 1 + ((config.api_thd[func_id] - 1) / ThdConfig.API_THD_STEP) + # 只存在负例 + else: + config.api_thd[func_id] = min(min(incon_ratio), config.api_thd[func_id]) + config.api_preheat[func_id] = False + + @staticmethod + def preheat(fuzz_ratio, is_consistent: bool, func_id): + """ + 预热，更新算子的比对阈值 + :param fuzz_ratio 扰动前后输出的比值 + :param is_consistent npu和cpu的输出结果比对是否一致 + :param func_id + """ + fuzz_ratio = fuzz_ratio[0] + # warmup + current_step = config.current_step + # 存储当前step算子调用的次数 + if func_id not in config.api_thd.keys(): + config.api_thd[func_id] = ThdConfig.PREHEAT_INITIAL_THD + # 存储当前step所有扰动前后的输出比值和对应npu、cpu的比对结果 + if current_step not in config.api_compare.keys(): + config.api_compare[current_step] = dict() + if func_id not in config.api_compare[current_step]: + config.api_compare[current_step][func_id] = list() + config.api_compare[current_step][func_id].append((fuzz_ratio, is_consistent)) + + preheat_counter.add_func_sampled_time(func_id) + if PreheatHandler.is_need_adjust_threshold(func_id): + try: + PreheatHandler.adjust_threshold(current_step, func_id) + except AttributeError as e: + logger.error(e) + api_used_statistic.update_api_thd() + + @staticmethod + def is_need_adjust_threshold(func_id) -> bool: + current_step = config.current_step + # 每一步应该采集的样本数 + sample_count_per_step = PreheatHandler.get_sample_count_per_step(func_id) + # 已经采集的样本数 + sampled_time = preheat_counter.get_func_sampled_time(func_id) + + res = sampled_time >= sample_count_per_step + if res: + logger.info(f'[preheat need_adjust_threshold] curr_step: {current_step}, ' + f'func_id: {func_id}, sampled_time: {sampled_time} / {sample_count_per_step}') + return res + + @staticmethod + def compare_npu_and_cpu(params: HandlerParams): + """ + 比较cpu和npu的执行结果是否满足上千双万的标准 + """ + args = convert_tensor_to_device(params.args) + kwargs = convert_tensor_to_device(params.kwargs) + cpu_result = params.origin_func(*args, **kwargs) + del args + del kwargs + origin_result = params.func_result.result_original + is_consistent = compare_seq(origin_result, cpu_result) + return is_consistent + + @staticmethod + def is_take_a_sample(func_id) -> bool: + """ + 是否采集 + 使用均匀采集策略（统计第0个step中api调用次数，均分到preheat_step步内进行采集） + """ + need_sample_set = PreheatHandler.get_need_sample_set(func_id) + # 当前是第几次调用 + curr_called_seq = preheat_counter.get_api_called_time(func_id) + # 是否采集 + res = curr_called_seq in need_sample_set + if res: + total_count = api_used_statistic.one_step_used_func_dict.get(func_id, 0) + logger.info(f'[preheat sample] curr_step: {config.current_step}, func_id: {func_id}, ' + f'curr_called_seq: {curr_called_seq} / {total_count}') + return res + + @staticmethod + def get_sample_count_per_step(func_id) -> int: + """ + 获取每一个step中应该采集的样本数 + """ + total_count = api_used_statistic.one_step_used_func_dict.get(func_id, 0) + preheat_step = device_settings[CCDMode.NPU].preheat_step + max_sample = device_settings[CCDMode.NPU].max_sample + return min(math.ceil(total_count / preheat_step), max_sample) + + @staticmethod + def get_need_sample_set(func_id) -> set: + """ + 需要采集的api调用时机（第几次调用）集合 + """ + # 一个step中当前api调用的总次数 + total_count = api_used_statistic.one_step_used_func_dict.get(func_id, 0) + # 每一步应该采集的样本数 + sample_count_per_step = PreheatHandler.get_sample_count_per_step(func_id) + need_sample_set = set() + preheat_step = device_settings[CCDMode.NPU].preheat_step + for i in range(1, sample_count_per_step + 1): + count = (preheat_step * (i - 1) + config.current_step) % total_count + if count == 0: + count = total_count + need_sample_set.add(count) + return need_sample_set + + def handle(self, params: HandlerParams) -> Any: + """ + :param params: cc_debugger.models.models.HandlerParams类 + :return: + """ + # 计算出当前api的2个ratio，y1：y2 + origin_output = params.func_result.result_original + fuzz_output = params.func_result.result_fuzzed + # 第0步统计各个api被调用的次数，用于预热采样 + if config.current_step == 0: + api_used_statistic.one_step_used_func_dict_statistic(func_id=params.func_id) + return origin_output + # 当前step中func_id被调用的次数+1，用于预热采样 + preheat_counter.add_api_called_time(self.func_id) + + if not self.is_take_a_sample(self.func_id): + return origin_output + + try: + # check_handle只针对浮点型 + if isinstance(fuzz_output, bool) or not Tools.check_output_type(fuzz_output): + return params.func_result.result_original + fuzz_ratio, npu_consistent = self.npu_compare(origin_output, fuzz_output, self.func_id) + if not npu_consistent: + self.save_unequal_result(params, fuzz_ratio, config.preheat_dir) + npu_and_cpu_cons = self.compare_npu_and_cpu(params) + logger.debug("{}: fuzz_ratio: {}, is_cons: {}", self.func_id, fuzz_ratio, npu_and_cpu_cons) + self.preheat(fuzz_ratio, npu_and_cpu_cons, self.func_id) + except Exception as e: + logger.error("api_name: {}, {}", self.func_id, e) + return params.func_result.result_original + logger.debug('[{}] preheat layer finished.', self.func_id) + # 和已知的范数进行比较 得到当前的最大值 + return params.func_result.result_original diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/relative_handler.py b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/relative_handler.py new file mode 100644 index 000000000..948d71b40 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/result_handlers/relative_handler.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import hashlib +import os +import time +from typing import Any +import numpy as np +import torch +from loguru import logger +from matplotlib import pyplot as plt +from cc_debugger.common.utils import Tools +from cc_debugger.core.config import config +from cc_debugger.core.const import FigConfig +from cc_debugger.models.models import HandlerParams + + +class RelativeHandler: + """ + 对相对误差进行处理 + + """ + + def __init__(self, func_id): + self.func_id = func_id + + # 开始绘制 + @staticmethod + def err_dist_hist(original_result, fuzz_result, dtype, func_id): + plt.clf() + # 计算Output Error Distrubution, Reshape to Dim=1 + m = torch.zeros_like(original_result) + scale = FigConfig.DTYPE_SCALE_THD.get(dtype) + res = torch.where(fuzz_result > FigConfig.LOWER_BOUND, + torch.div(original_result - fuzz_result, original_result) * scale, m).reshape(-1) + # 直方图上下界,分隔的段数,Y轴类型 + bound_max = FigConfig.MAX + if FigConfig.ABS_MODE: + res = torch.abs(res) + bound_min = 0 + else: + bound_min = - bound_max + cnt = FigConfig.CNT * (bound_max - bound_min) + if FigConfig.Y_LOG: + plt.yscale('log') + sort = torch.histc(res, cnt, bound_min, bound_max).cpu() + plt.plot(np.linspace(bound_min, bound_max, cnt), sort) + + # 保存位置save_root/func_name/func_id.png + plt.xlabel("Rel-Error * " + str(scale)) + plt.ylabel("Count") + file_name = f"{str(original_result)}_{time.time()}" + file_name = hashlib.sha1(file_name.encode('utf-8')).hexdigest() + '.png' + file_path = config.fig_dir.joinpath(str(dtype)) + fig_path = file_path.joinpath(func_id).joinpath(file_name) + os.makedirs(fig_path.parent, exist_ok=True) + plt.savefig(fig_path) + + def handle(self, params: HandlerParams) -> Any: + """ + :param params: cc_debugger.models.models.HandlerParams类 + :return: + """ + # 计算出当前api的2个ratio，y1：y2 + ans1 = params.func_result.result_original + ans2 = params.func_result.result_fuzzed + try: + # check_handle只针对浮点型 + input_dtype = Tools.get_first_tensor(params.args).dtype + if isinstance(ans1, (list, tuple)): + self.err_dist_hist(ans1[0], ans2[0].to(ans1[0].dtype), input_dtype, params.func_id) + elif isinstance(ans1, torch.Tensor): + self.err_dist_hist(ans1, ans2.to(ans1.dtype), input_dtype, params.func_id) + except Exception as e: + logger.error(e) + return params.func_result.result_original + logger.debug('[{}] relative layer finished.', self.func_id) + # 和已知的范数进行比较 得到当前的最大值 + return params.func_result.result_original diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_base.py b/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_base.py new file mode 100644 index 000000000..91bc0b2a1 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_base.py @@ -0,0 +1,95 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import pickle +from dataclasses import dataclass +from io import BytesIO +from typing import IO, Any +from urllib.parse import urljoin + +import requests +import torch +from cc_debugger.core.config import base_settings, device_settings +from cc_debugger.core.enums import CCDMode +from cc_debugger.run_api.run_case import RunCaseFactory + +try: + import torch_npu +except ImportError: + pass + + +@dataclass +class UploadData: + api_name: str + test_mode: str + + +@dataclass +class InputAndOutputFiles: + up_input_file: IO[Any] + up_output_file: IO[Any] + + def to_dict(self): + return { + 'up_input_file': self.up_input_file, + 'up_output_file': self.up_output_file + } + + +class BaseRun: + def __init__(self, api_name, data: dict, device): + self.api_name = api_name + self.data = data + self.device = device + + def convert_tensor_dtype(self, tensor_seq, origin_dtype: torch.dtype, target_dtype: torch.dtype): + """ + cpu can't deal with fp16, but can deal with bf16 + If the dtype of the tensor is float16, it is changed to float32 + :return: + """ + if isinstance(tensor_seq, torch.Tensor) and tensor_seq.dtype == origin_dtype: + return tensor_seq.to(target_dtype) + if isinstance(tensor_seq, dict): + return {k: self.convert_tensor_dtype(v, origin_dtype, target_dtype) for k, v in tensor_seq.items()} + if isinstance(tensor_seq, (tuple, list)): + return type(tensor_seq)([self.convert_tensor_dtype(v, origin_dtype, target_dtype) for v in tensor_seq]) + return tensor_seq + + def run(self): + run_case = RunCaseFactory.create_run_case() + rc = run_case(self.api_name, self.data, self.device) + return rc.run_on_device() + + def cpu_run(self): + """ + use the same input data with npu, but convert fp16->fp32 for cpu can not calculate fp16 + :return: + """ + input_data = self.convert_tensor_dtype(self.data, torch.float16, torch.float32) + run_case = RunCaseFactory.create_run_case() + rc = run_case(self.api_name, input_data, CCDMode.CPU.value) + res = rc.run_on_device() + return res + + +class RemoteRun: + + @staticmethod + def run_remote_result(name, req_data, api_url): + """ + get the remote server run result + :param name: + :param req_data: + :param api_url: + :return: + """ + _server_url = urljoin(device_settings.get(CCDMode.GPU).server_url, api_url) + with BytesIO(pickle.dumps(req_data, protocol=4)) as fb: + r: requests.Response = requests.post(url=_server_url, data={"api_name": name, + "test_mode": base_settings.mode.value}, + files={"up_input_file": ("input.pickle", fb)}) + if r.status_code == 200: + return pickle.loads(r.content) + raise ValueError(f"Failed GPU RUN {name}") diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_case.py b/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_case.py new file mode 100644 index 000000000..03c1fb889 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/run_api/run_case.py @@ -0,0 +1,88 @@ +# !/usr/bin/python3.7 +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +from typing import Dict + +import torch +from loguru import logger + +import cc_debugger.common.utils as utils +from cc_debugger.core.enums import ModeType, InputKey + + +class RunCase: + def __init__(self, api_name: str, data: dict, device): + self.api_name = api_name + self.args = data.get(InputKey.ARGS.value) + self.kwargs = data.get(InputKey.KWARGS.value) + self.device = device + self.load_data_to_device() + + def load_data_to_device(self): + self.args = utils.convert_tensor_to_device(self.args, self.device) + self.kwargs = utils.convert_tensor_to_device(self.kwargs, self.device) + + def run_on_device(self): + api = self.api_name + if isinstance(api, str): + api = eval(api) + return api(*self.args, **self.kwargs) + + +class BackwardRunCase(RunCase): + def __init__(self, api_name: str, data: Dict, device): + super().__init__(api_name, data, device) + self.grad_outputs = utils.convert_tensor_to_device(InputKey.GRAD.value, device) + self.needs_input_grad = data.get(InputKey.NEED_GRAD.value) + self.grad_inputs = [] + + def set_tensors_required_grad(self, inputs): + if isinstance(inputs, torch.Tensor) and "float" in str(inputs.dtype): + inputs.requires_grad = True + self.grad_inputs.append(inputs) + return inputs + if isinstance(inputs, dict): + return {k: self.set_tensors_required_grad(v) for k, v in inputs.items()} + if isinstance(inputs, (tuple, list)): + return [self.set_tensors_required_grad(v) for v in inputs] + return inputs + + def set_real_tensors_requires_grad(self): + for i, v in enumerate(self.needs_input_grad): + if v: + self.args[i].requires_grad = True + self.grad_inputs.append(self.args[i]) + return self.args + + def run_on_device(self): + if self.grad_outputs: + self.args = self.set_real_tensors_requires_grad() + else: + self.args = self.set_tensors_required_grad(self.args) + self.kwargs = self.set_tensors_required_grad(self.kwargs) + output_data = super().run_on_device() + if self.grad_outputs is None: + logger.info("no grad_outputs get, use torch ones_like to generate") + self.grad_outputs = torch.ones_like(output_data.detach()) + # backward only once + if "torch.return_types" in str(type(output_data)): + output_data = output_data.values + args_grad = torch.autograd.grad(output_data, self.grad_inputs, self.grad_outputs) + return args_grad + + +class RunCaseFactory: + run_test_map = { + ModeType.BACKWARD.value: BackwardRunCase, + ModeType.FORWARD.value: RunCase + } + + @staticmethod + def create_run_case(test_mode: ModeType = None): + if not test_mode: + test_mode = ModeType.FORWARD.value + run_test_class = RunCaseFactory.run_test_map.get(test_mode) + if run_test_class: + return run_test_class + else: + raise ValueError(f"Invalid run test mode: {test_mode}") diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/__init__.py b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/__init__.py new file mode 100644 index 000000000..c39257b25 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_preheat_counter.py b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_preheat_counter.py new file mode 100644 index 000000000..973025972 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_preheat_counter.py @@ -0,0 +1,28 @@ +class PreheatCounter: + def __init__(self): + # 当前预热step，某个api已经被调用的次数 + self.func_called_time: dict = dict() + # 当前预热step，某个api已经被采样的次数 + self.func_sampled_time: dict = dict() + def reset(self): + self.__init__() + + def add_api_called_time(self, func_id: str): + if func_id not in self.func_called_time: + self.func_called_time[func_id] = 0 + self.func_called_time[func_id] += 1 + + def get_api_called_time(self, func_id: str) -> int: + return self.func_called_time.get(func_id, 0) + + def add_func_sampled_time(self, func_id: str): + if func_id not in self.func_sampled_time: + self.func_sampled_time[func_id] = 0 + self.func_sampled_time[func_id] += 1 + + def get_func_sampled_time(self, func_id: str) -> int: + return self.func_sampled_time.get(func_id, 0) + +# 预热阶段每一步需要的计数器 +# 注意：需要在每次迭代时重置 +preheat_counter = PreheatCounter() \ No newline at end of file diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_unused.py b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_unused.py new file mode 100644 index 000000000..120dbe43b --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_unused.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import argparse +import json +import os +import stat +from pathlib import Path + + +def get_unused_api(fuzz_path: Path, fuzz_result_path: Path, store_path: Path): + """ + 获取模型中所用到的api和未使用fuzz扰动技术的api，并将结果存放至txt + :param fuzz_path: 扰动算子列表 + :param fuzz_result_path：存放扰动结果的文件地址 + :param store_path: 存放结果的路径 + """ + fuzz_apis = list() + with open(fuzz_path, 'r') as f: + for line in f: + line = line.strip() + fuzz_apis.append(line.strip()) + with open(fuzz_result_path, 'r') as f: + fuzz_result = json.load(f) + used_api_dict = fuzz_result.get("used_api_dict") + used_apis = used_api_dict.keys() + flags = os.O_RDWR | os.O_TRUNC | os.O_CREAT + modes = stat.S_IRUSR | stat.S_IWUSR + os.makedirs(store_path, exist_ok=True) + with os.fdopen(os.open(store_path.joinpath("model_apis.txt"), flags, modes), 'w') as f1, \ + os.fdopen(os.open(store_path.joinpath("not_fuzz_apis.txt"), flags, modes), 'w') as f2: + for api in used_apis: + f1.write(api + "\n") + if api not in fuzz_apis: + f2.write(api + "\n") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + current_path = Path(__file__).parent + parser.add_argument('--fuzz_path', default=current_path.joinpath("../torch_api.txt"), type=Path, + help='fuzz api path') + parser.add_argument('--result_path', type=Path, help='result data directory') + parser.add_argument('--store_path', default=Path("./result"), type=Path, help='store result directory') + + args = parser.parse_args() + get_unused_api(args.fuzz_path, args.result_path, args.store_path) diff --git a/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_used.py b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_used.py new file mode 100644 index 000000000..766b416ff --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/cc_debugger/statistics/api_used.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +import json +import os +from pathlib import Path +from typing import Dict + +from loguru import logger + +from cc_debugger.core.config import config +from cc_debugger.core.const import Constant + + +class ApiUsed: + """ + torch算子使用情况统计 + + """ + + def __init__(self): + # 触发插桩的api + self.called_api_dict: Dict = dict() + # 正常流程结束的api + self.normal_process_api_dict: Dict = dict() + # 异常结束的api + self.exception_api_dict: Dict = dict() + # 不满足条件而退出的api(堆栈限制) + self.not_stack_api_dict: Dict = dict() + # 入参不满足要求的api + self.not_input_param_api_dict: Dict = dict() + # 入参未添加扰动的api + self.not_added_api_dict: Dict = dict() + # 模型中所有用到的api + self.used_api_dict: Dict = dict() + # 模型中所有调用函数 + self.used_fuc_dict: Dict = dict() + # 所有算子的阈值 + self.api_thd: Dict = dict() + # 一个step调用的api（第0步） + self.one_step_used_func_dict: Dict = dict() + # 未被预热的api + self.func_no_preheat_list: list = [] + + @staticmethod + def totalization(operand: Dict, func_id: str): + """ + 对api被调用次数进行累加操作 + + :param operand: + :param func_id: + :return: + """ + api_count = operand.get(func_id) + if api_count: + operand[func_id] += 1 + else: + operand[func_id] = 1 + + def called_statistic(self, func_id: str): + """ + 统计算子进入装饰器的次数 + + :param func_id: + :return: + """ + self.totalization(operand=self.called_api_dict, func_id=func_id) + + def normal_process_statistic(self, func_id: str): + """ + 正常流程结束的算子统计 + + :param func_id: + :return: + """ + self.totalization(operand=self.normal_process_api_dict, func_id=func_id) + + def exception_statistic(self, func_id: str): + """ + 异常结束的算子统计 + + :param func_id: + :return: + """ + self.totalization(operand=self.exception_api_dict, func_id=func_id) + + def not_stack_statistic(self, func_id: str): + """ + 堆栈限制而退出的算子统计 + + :param func_id: + :return: + """ + self.totalization(operand=self.not_stack_api_dict, func_id=func_id) + + def not_input_param_statistic(self, func_id: str): + """ + 不符合要求的类型入参的算子统计 + + :param func_id: + :return: + """ + self.totalization(operand=self.not_input_param_api_dict, func_id=func_id) + + def not_added_statistic(self, func_id: str): + """ + 未添加扰动而退出的算子统计 + + :param func_id: + :return: + """ + self.totalization(operand=self.not_added_api_dict, func_id=func_id) + + def used_api_statistic(self, func_id: str): + """ + 模型中所有用到的算子统计 + + :param func_id: + :return: + """ + self.totalization(operand=self.used_api_dict, func_id=func_id) + + def used_func_statistic(self, funcs: set): + """ + 模型中所有用到的算子统计 + + :param funcs: + :return: + """ + self.totalization(operand=self.used_fuc_dict, func_id=str(funcs)) + + def update_api_thd(self): + """ + 记录所有预热阶段算子最终的阈值 + """ + self.api_thd = config.api_thd + + def one_step_used_func_dict_statistic(self, func_id: str): + """ + 一个step调用的算子数量统计 + """ + self.totalization(operand=self.one_step_used_func_dict, func_id=func_id) + + def func_no_preheat_list_statistic(self) -> list: + """ + 未被预热的算子统计 + """ + preheated = self.one_step_used_func_dict.keys() + no_preheat: list = [] + for func_id in config.test_apis: + if func_id not in preheated: + no_preheat.append(func_id) + self.func_no_preheat_list = no_preheat + return no_preheat + + def save_to_file(self): + """ + 将数据写入文件 + :return + """ + api_used_statistic_file: Path = config.statistic_dir.joinpath('api_used_statistic.json') + os.makedirs(api_used_statistic_file.parent, exist_ok=True) + try: + with os.fdopen(os.open(api_used_statistic_file, Constant.FILE_FLAGS, Constant.FILE_MODES), 'w') as file: + json.dump(self.__dict__, file, indent=2) + except Exception as e: + logger.error(e) + + +api_used_statistic = ApiUsed() diff --git a/debug/accuracy_tools/CCDebugger/pyproject.toml b/debug/accuracy_tools/CCDebugger/pyproject.toml new file mode 100644 index 000000000..221f4f967 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/pyproject.toml @@ -0,0 +1,28 @@ +[tool.poetry] +name = "cc_debugger" +version = "1.0.0" +description = "cross-server cross-architecture learning debugger" +authors = ["z00576831"] +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.7.5,<3.11" +loguru = "^0.7.2" +cleo = "^2.1.0" +pydantic-settings = "2.0.3" +python-dotenv = "0.21.1" +torch = "1.11.0" +pandas = "1.3.5" +matplotlib = "3.5.3" + +[tool.poetry.dev-dependencies] +pytest = "^5.2" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[[tool.poetry.source]] +name = 'mirrors.tools.huawei.com' +url = "http://mirrors.tools.huawei.com/pypi/simple" +priority = "default" diff --git a/debug/accuracy_tools/CCDebugger/tests/test_backward.py b/debug/accuracy_tools/CCDebugger/tests/test_backward.py new file mode 100644 index 000000000..7a8473b8b --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/tests/test_backward.py @@ -0,0 +1,91 @@ +import json +import pickle +import shutil +from pathlib import Path +from unittest import mock + +import torch +from cc_debugger.core.config import device_settings, base_settings, config +from cc_debugger.core.enums import PerturbationModeType, CCDMode, HandlerType +from cc_debugger.decorator.debugger import decorate_backward_function + + +class TestBackward: + current_path = Path(__file__).parent + output_path = current_path.parent.joinpath("ccd_output") + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.output_path, ignore_errors=True) + + def test_backward_cpu(self): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.ADD_NOISE + base_settings.handler_type = HandlerType.CHECK + config.is_running = True + config.test_apis = ["torch.nn.functional.linear"] + base_settings.ccd_mode = {CCDMode.CPU: 0} + base_settings.ccd_flag = True + inputs = torch.randn(3, 5, requires_grad=True).to("npu:0") # 输入张量 + weight = torch.randn(2, 5, requires_grad=True).to("npu:0") # 权重张量 + output = decorate_backward_function(torch.nn.functional.linear, + func_id="torch.nn.functional.linear")(inputs, weight) + output.backward(torch.ones_like(output)) + api_used_statistic = self.output_path.joinpath("statistic/api_used_statistic.json") + with open(api_used_statistic, 'r') as f: + data = json.load(f) + assert data.get("normal_process_api_dict").get("torch.nn.functional.linear") == 1 + assert data.get("exception_api_dict").get("torch.nn.functional.linear") is None + + def test_backward_npu(self): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + base_settings.ccd_mode = {CCDMode.NPU: 0} + base_settings.ccd_flag = True + config.test_apis = ["torch.mul"] + config.is_running = True + x = torch.randn(2, 3, requires_grad=True, device="npu:0") + y = torch.randn(2, 3, requires_grad=True, device="npu:0") + z = decorate_backward_function(torch.mul, func_id="torch.mul")(x, y) + z.backward(torch.ones_like(z)) + api_used_statistic = self.output_path.joinpath("statistic/api_used_statistic.json") + with open(api_used_statistic, 'r') as f: + data = json.load(f) + assert data.get("normal_process_api_dict").get("torch.mul") == 1 + assert data.get("exception_api_dict").get("torch.mul") is None + + @mock.patch("requests.post") + def test_backward_gpu(self, request_post): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + base_settings.handler_type = HandlerType.FIX + config.is_running = True + config.test_apis = ["torch.sin"] + res_grad = [torch.as_tensor([[0.8272, 0.2355, -0.9569], [-0.6181, -0.9111, -0.9063]])] + _m_response = mock.Mock(status_code=200, content=pickle.dumps(res_grad)) + request_post.return_value = _m_response + base_settings.ccd_mode = {CCDMode.GPU: 0} + base_settings.ccd_flag = True + a = torch.as_tensor([[0.9740, 0.2377, -1.2762], + [-0.6663, -1.1459, -1.1344]], dtype=torch.float32, device="npu:0") + a.requires_grad = True + b = decorate_backward_function(torch.sin)(a) + b.backward(torch.ones_like(b)) + assert torch.allclose(a.grad, res_grad[0].to("npu:0")) + + @mock.patch("requests.post") + def test_backward_mix_machines(self, request_post): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + base_settings.handler_type = HandlerType.FIX + config.is_running = True + config.test_apis = ["torch.sin"] + res_grad = [torch.as_tensor([[0.8272, 0.2355, -0.9569], [-0.6181, -0.9111, -0.9063]])] + npu_grad = [torch.as_tensor([[0.5619955063, 0.9718821049, 0.2903535366], + [0.7861139178, 0.4122263193, 0.4226762652]])] + _m_response = mock.Mock(status_code=200, content=pickle.dumps(res_grad)) + request_post.return_value = _m_response + base_settings.ccd_mode = {CCDMode.GPU: 0, CCDMode.NPU: 0, CCDMode.CPU: 0} + base_settings.ccd_flag = True + a = torch.as_tensor([[0.9740, 0.2377, -1.2762], + [-0.6663, -1.1459, -1.1344]], dtype=torch.float32, device="npu:0") + a.requires_grad = True + b = decorate_backward_function(torch.sin)(a) + b.backward(torch.ones_like(b)) + assert torch.allclose(a.grad, npu_grad[0].to("npu:0")) diff --git a/debug/accuracy_tools/CCDebugger/tests/test_distributed_apis.py b/debug/accuracy_tools/CCDebugger/tests/test_distributed_apis.py new file mode 100644 index 000000000..8abbe3f64 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/tests/test_distributed_apis.py @@ -0,0 +1,74 @@ +import json +import os +import shutil +from pathlib import Path +import torch +import torch_npu +from torch.distributed import init_process_group, destroy_process_group +import torch.multiprocessing as mp +from cc_debugger.core.config import config, base_settings, device_settings +from cc_debugger.core.enums import HandlerType, CCDMode, PerturbationModeType +from cc_debugger.decorator.debugger import decorate_forward_function +from cc_debugger.decorator.dec_forward import NpuForward +from cc_debugger.models.models import HandlerParams, FuncResult + + +class TestDistributedApis: + current_path = Path(__file__).parent + output_path = current_path.parent.joinpath("ccd_output") + + @staticmethod + def ddp_setup(rank, world_size): + os.environ['MASTER_ADDR'] = 'localhost' + os.environ['MASTER_PORT'] = '12355' + init_process_group(backend='hccl', rank=rank, world_size=world_size) + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.output_path, ignore_errors=True) + + def use_distributed(self, rank, world_size): + config.is_running = True + torch.npu.set_device(f"npu:{rank}") + self.ddp_setup(rank, world_size) + rank = torch.distributed.get_rank() + tensor_size = 10 * 10 * 10 + inputs = torch.randn(tensor_size).to(f"npu:{rank}") + all_reduce = decorate_forward_function(torch.distributed.all_reduce, + func_id="test_torch.distributed.all_reduce") + all_reduce(inputs, op=torch.distributed.ReduceOp.SUM) + if world_size != -1: + destroy_process_group() + + def test_get_compare_data(self): + base_settings.handler_type = HandlerType.FIX + base_settings.ccd_mode = {CCDMode.NPU: 0} + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + config.is_running = True + handler_params = HandlerParams() + handler_params.func_id = "torch.add" + args = (torch.as_tensor([0.01, 0.02], dtype=torch.float16)) + origin_res = torch.as_tensor([0.12, 0.13], dtype=torch.float16) + fuzz_res = torch.as_tensor([0.11, 0.12], dtype=torch.float16) + handler_params.func_result = FuncResult(result_original=origin_res, result_fuzzed=fuzz_res) + handler_params.fuzzed_value = (torch.as_tensor([0.02, 0.03], dtype=torch.float16), {}) + handler_params.func_id = "torch.distributed.all_reduce" + handler_params.args = args + npu_forward = NpuForward(handler_params) + npu_forward.get_compare_data() + assert torch.allclose(handler_params.func_result.result_original, args) + assert torch.allclose(handler_params.func_result.result_fuzzed, handler_params.fuzzed_value[0]) + + def test_all_reduce(self): + base_settings.handler_type = HandlerType.FIX + base_settings.ccd_mode = {CCDMode.NPU: 0} + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + config.is_running = True + world_size = 2 + mp.spawn(self.use_distributed, args=(world_size,), nprocs=world_size) + api_used_statistic = self.output_path.joinpath("statistic/api_used_statistic.json") + with open(api_used_statistic, 'r') as f: + data = json.load(f) + assert data.get("normal_process_api_dict").get("test_torch.distributed.all_reduce") == 1 + + diff --git a/debug/accuracy_tools/CCDebugger/tests/test_forward.py b/debug/accuracy_tools/CCDebugger/tests/test_forward.py new file mode 100644 index 000000000..2af62fc58 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/tests/test_forward.py @@ -0,0 +1,91 @@ +import json +import pickle +import shutil +from pathlib import Path +from unittest import mock + +import torch +from cc_debugger.core.config import device_settings, base_settings, config +from cc_debugger.core.enums import PerturbationModeType, CCDMode, HandlerType +from cc_debugger.decorator.debugger import decorate_forward_function + + +class TestForward: + current_path = Path(__file__).parent + output_path = current_path.parent.joinpath("ccd_output") + + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.output_path, ignore_errors=True) + + def test_forward_cpu(self): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.ADD_NOISE + base_settings.handler_type = HandlerType.CHECK + config.is_running = True + config.test_apis = ["torch.Tensor.mul_"] + base_settings.ccd_mode = {CCDMode.CPU: 0} + base_settings.ccd_flag = True + inputs = torch.randn(2, 5).to("npu:0") # 输入张量 + weight = torch.randn(2, 5).to("npu:0") # 权重张量 + decorate_forward_function(torch.Tensor.mul_, func_id="torch.Tensor.mul_")(inputs, weight) + api_used_statistic = self.output_path.joinpath("statistic/api_used_statistic.json") + with open(api_used_statistic, 'r') as f: + data = json.load(f) + assert data.get("normal_process_api_dict").get("torch.Tensor.mul_") == 1 + assert data.get("exception_api_dict").get("torch.Tensor.mul_") is None + + def test_forward_npu(self): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + base_settings.handler_type = HandlerType.CHECK + base_settings.ccd_mode = {CCDMode.NPU: 0} + base_settings.ccd_flag = True + config.test_apis = ["torch.Tensor.sum"] + config.is_running = True + x = torch.randn(2, 3, device="npu:0") + y = torch.randn(2, 3, device="npu:0") + decorate_forward_function(torch.mul, func_id="torch.Tensor.sum")(x, y) + api_used_statistic = self.output_path.joinpath("statistic/api_used_statistic.json") + with open(api_used_statistic, 'r') as f: + data = json.load(f) + assert data.get("normal_process_api_dict").get("torch.Tensor.sum") == 1 + assert data.get("exception_api_dict").get("torch.Tensor.sum") is None + + @mock.patch("requests.post") + def test_forward_gpu(self, request_post): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + base_settings.handler_type = HandlerType.FIX + config.is_running = True + config.test_apis = ["torch.cat"] + gpu_output = torch.as_tensor([[0.5424, 0.2897, 0.0126], + [0.9236, 0.9488, 0.3470], + [0.9020, 0.0070, 0.2939], + [0.5599, 0.6294, 0.8289]]) + _m_response = mock.Mock(status_code=200, content=pickle.dumps(gpu_output)) + request_post.return_value = _m_response + base_settings.ccd_mode = {CCDMode.GPU: 0} + base_settings.ccd_flag = True + a = torch.as_tensor([[0.5424, 0.0897, 0.0126], + [0.9236, 0.9488, 0.3470]], dtype=torch.float32, device="npu:0") + b = torch.as_tensor([[0.8020, 0.0070, 0.2939], + [0.5599, 0.6294, 0.8289]], dtype=torch.float32, device="npu:0") + b = decorate_forward_function(torch.cat)([a, b]) + assert torch.allclose(b, gpu_output.to("npu:0")) + + @mock.patch("requests.post") + def test_forward_mix_machines(self, request_post): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + base_settings.handler_type = HandlerType.FIX + config.is_running = True + config.test_apis = ["torch.cos"] + res_grad = torch.as_tensor([[0.8272, 0.2355, -0.9569], [-0.6181, -0.9111, -0.9063]]) + npu_grad = torch.as_tensor([[0.5619955063, 0.9718821645, 0.2903535664], + [0.7861139774, 0.4122262895, 0.4226762652]]) + _m_response = mock.Mock(status_code=200, content=pickle.dumps(res_grad)) + request_post.return_value = _m_response + base_settings.ccd_mode = {CCDMode.GPU: 0, CCDMode.NPU: 0, CCDMode.CPU: 0} + base_settings.ccd_flag = True + a = torch.as_tensor([[0.9740, 0.2377, -1.2762], + [-0.6663, -1.1459, -1.1344]], dtype=torch.float32, device="npu:0") + b = decorate_forward_function(torch.cos)(a) + assert torch.allclose(b, npu_grad.to("npu:0")) diff --git a/debug/accuracy_tools/CCDebugger/tests/test_input_bound.py b/debug/accuracy_tools/CCDebugger/tests/test_input_bound.py new file mode 100644 index 000000000..13336c5db --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/tests/test_input_bound.py @@ -0,0 +1,52 @@ +import shutil +from pathlib import Path +import torch +from cc_debugger.core.config import device_settings +from cc_debugger.core.enums import PerturbationModeType, CCDMode +from cc_debugger.layers.fuzz_layer import FuzzLayer + + +class TestLowerBoundAndNoise: + current_path = Path(__file__).parent + output_path = current_path.parent.joinpath("ccd_output") + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.output_path, ignore_errors=True) + + def test_lower_bound(self): + device_settings[CCDMode.NPU].pert_type = PerturbationModeType.ADD_NOISE + inputs = torch.as_tensor(1e-7, dtype=torch.float16) + fuzz_layer = FuzzLayer(func_id="test") + is_added, fuzzed_value = fuzz_layer(inputs) + assert is_added is False + inputs = torch.as_tensor([1e-7, 1e-2], dtype=torch.float16) + is_added, fuzzed_value = fuzz_layer(inputs) + assert is_added + assert fuzzed_value[0] == 1e-7 + assert fuzzed_value[1] == 1e-2 + 1e-6 + + def test_tensor_list(self): + device_settings[CCDMode.NPU].pert_type = PerturbationModeType.ADD_NOISE + + a = torch.as_tensor(3, dtype=torch.int32) + b = torch.as_tensor([1e-9, 1e-2], dtype=torch.float32) + inputs = [a, b] + fuzz_layer = FuzzLayer(func_id="test") + is_added, fuzzed_value = fuzz_layer(inputs) + assert is_added + assert fuzzed_value[0].item() == 3 + assert fuzzed_value[1][0] == 1e-9 + assert fuzzed_value[1][1] == 1e-2 + 1e-8 + + def test_tensor_dict(self): + device_settings[CCDMode.NPU].pert_type = PerturbationModeType.ADD_NOISE + a = torch.as_tensor(3, dtype=torch.int32) + b = torch.as_tensor([1e-9, 1e-2], dtype=torch.float32) + inputs = {"a": a, "b": b} + fuzz_layer = FuzzLayer(func_id="test") + is_added, fuzzed_value = fuzz_layer(inputs) + assert is_added + assert fuzzed_value["a"].item() == 3 + assert fuzzed_value["b"][0] == 1e-9 + assert fuzzed_value["b"][1] == 1e-2 + 1e-8 diff --git a/debug/accuracy_tools/CCDebugger/tests/test_perturbation.py b/debug/accuracy_tools/CCDebugger/tests/test_perturbation.py new file mode 100644 index 000000000..cd59ac21b --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/tests/test_perturbation.py @@ -0,0 +1,51 @@ +import shutil +from pathlib import Path +import torch +from cc_debugger.core.config import device_settings +from cc_debugger.core.enums import CCDMode, PerturbationModeType +from cc_debugger.layers.fuzz_layer import FuzzLayer + + +class TestPertType: + current_path = Path(__file__).parent + output_path = current_path.parent.joinpath("ccd_output") + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.output_path, ignore_errors=True) + + def test_improve_precision(self): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.IMPROVE_PRECISION + a = torch.rand(2, 3, dtype=torch.bfloat16) + b = torch.rand(2, 3, dtype=torch.float16) + c = torch.rand(2, 3, dtype=torch.float32) + d = torch.rand(2, 3, dtype=torch.float64) + f = torch.rand(2, 3, dtype=torch.float64).to(torch.int32) + inputs = [a, b, {"c": c, "d": d}, f] + fuzz_layer = FuzzLayer(func_id="test") + # 得到加了扰动之后的值 + is_added, fuzzed_value = fuzz_layer(inputs) + assert is_added + assert fuzzed_value[0].dtype == torch.float32 + assert fuzzed_value[1].dtype == torch.float32 + assert fuzzed_value[2]["c"].dtype == torch.float64 + assert fuzzed_value[2]["d"].dtype == torch.float64 + assert fuzzed_value[3].dtype == torch.int32 + + def test_no_change(self): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.NO_CHANGE + inputs = torch.as_tensor([1e-9, 1e-2], dtype=torch.float32) + fuzz_layer = FuzzLayer(func_id="test") + is_added, fuzzed_value = fuzz_layer(inputs) + assert is_added + assert fuzzed_value[0] == 1e-9 + assert fuzzed_value[1] == 1e-2 + + def test_change_value(self): + device_settings.get(CCDMode.NPU).pert_type = PerturbationModeType.CHANGE_VALUE + inputs = torch.as_tensor([1e-9, 1e-2], dtype=torch.float32) + fuzz_layer = FuzzLayer(func_id="test") + is_added, fuzzed_value = fuzz_layer(inputs) + assert is_added + assert fuzzed_value[0] == 1e-2 + assert fuzzed_value[1] == 1e-9 diff --git a/debug/accuracy_tools/CCDebugger/tests/test_preheat.py b/debug/accuracy_tools/CCDebugger/tests/test_preheat.py new file mode 100644 index 000000000..be8e85833 --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/tests/test_preheat.py @@ -0,0 +1,43 @@ +import shutil +from pathlib import Path +import torch +from torch.utils.data import TensorDataset, DataLoader + +from cc_debugger.core.config import config, base_settings, device_settings +from cc_debugger.core.enums import CCDMode +from cc_debugger.decorator.debugger import decorate_backward_function, decorate_forward_function +from cc_debugger.main import iter_tracer + + +class TestPreheat: + current_path = Path(__file__).parent + output_path = current_path.parent.joinpath("ccd_output") + + @classmethod + def setup_class(cls): + base_settings.ccd_flag = True + base_settings.ccd_mode = {CCDMode.NPU: 0} + device_settings[CCDMode.NPU].is_preheat = True + config.test_apis = ["torch.add"] + torch.utils.data.dataloader._BaseDataLoaderIter.__next__ = iter_tracer(torch.utils.data.dataloader. + _BaseDataLoaderIter.__next__) + + def sum_preheat(self, sum_api): + x = torch.linspace(1, 10, 40, requires_grad=True) # x data (torch tensor) + y = torch.linspace(10, 1, 40, requires_grad=True) # y data (torch tensor) + dataset = TensorDataset(x, y) + loader = DataLoader(dataset, batch_size=2) + config.test_apis = ["torch.add"] + for step, y in enumerate(loader): + result = sum_api(*y) + result.backward(torch.ones_like(result)) + assert config.api_thd.get("torch.add") < 2.05 + shutil.rmtree(self.output_path, ignore_errors=True) + + def test_forward_sum_preheat(self): + sum_api = decorate_forward_function(torch.add, func_id="torch.add") + self.sum_preheat(sum_api) + + def test_backward_sum_preheat(self): + sum_api = decorate_backward_function(torch.add, func_id="torch.add") + self.sum_preheat(sum_api) diff --git a/debug/accuracy_tools/CCDebugger/tests/test_result_handler.py b/debug/accuracy_tools/CCDebugger/tests/test_result_handler.py new file mode 100644 index 000000000..3694c847d --- /dev/null +++ b/debug/accuracy_tools/CCDebugger/tests/test_result_handler.py @@ -0,0 +1,44 @@ +import shutil +from pathlib import Path +import torch +import torch_npu + +from cc_debugger.core.config import base_settings, config +from cc_debugger.core.enums import CCDMode, HandlerType +from cc_debugger.decorator.dec_forward import OtherForward +from cc_debugger.models.models import HandlerParams, FuncResult + + +class TestResultHandler: + current_path = Path(__file__).parent + output_path = current_path.parent.joinpath("ccd_output") + + @classmethod + def teardown_class(cls): + shutil.rmtree(cls.output_path, ignore_errors=True) + + def test_result_handler(self): + origin_inputs = [torch.as_tensor([3.01, 3.02], dtype=torch.float16, device="npu:0"), + torch.as_tensor([0.02, 0.02], dtype=torch.float16, device="npu:0")] + fuzz_inputs = [(i + 0.002).to(torch.float32).to("cpu") for i in origin_inputs] + origin_res = torch.add(*origin_inputs) + fuzz_res = torch.add(*fuzz_inputs) + handler_params = HandlerParams() + handler_params.func_id = "torch.add" + handler_params.func_result = FuncResult(result_original=origin_res, result_fuzzed=fuzz_res) + handler_params.device = CCDMode.CPU + # check + base_settings.handler_type = HandlerType.CHECK + other_forward = OtherForward(handler_params) + result = other_forward.deal_fuzzed_and_original_result() + check_path = config.check_dir + file_count = len(list(check_path.glob("**/*.p"))) + assert file_count == 3 + # fix + base_settings.handler_type = HandlerType.FIX + other_forward = OtherForward(handler_params) + result = other_forward.deal_fuzzed_and_original_result() + assert result.dtype == torch.float16 + assert result.device == origin_inputs[0].device + assert torch.allclose(result, fuzz_res.to(torch.float16).to("npu:0")) + -- Gitee