From b5d02c63c0c32282e9bbd267d62955208f76a422 Mon Sep 17 00:00:00 2001 From: 13611516375 Date: Thu, 16 Jan 2025 11:23:45 +0000 Subject: [PATCH 01/17] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=B8=B8=E8=A7=81?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E6=A0=B7=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 60 ++ .../AclTensorFreeFail/AclNNInvocation/run.sh | 116 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 26 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 68 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 164 ++++ .../AclNNInvocation/src/op_runner.cpp | 485 +++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../AddCustom/CMakeLists.txt | 69 ++ .../AddCustom/CMakePresets.json | 63 ++ .../AclTensorFreeFail/AddCustom/build.sh | 37 + .../AddCustom/cmake/config.cmake | 25 + .../AddCustom/cmake/func.cmake | 228 +++++ .../AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 73 ++ .../AddCustom/op_host/add_custom_tiling.h | 22 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 96 ++ .../AddCustom/scripts/install.sh | 318 +++++++ .../AddCustom/scripts/upgrade.sh | 151 ++++ .../AddKernelInvocation/CMakeLists.txt | 10 + .../AddKernelInvocation/README.md | 81 ++ .../AddKernelInvocation/add_custom.cpp | 91 ++ .../cmake/Modules/CMakeCCECompiler.cmake.in | 5 + .../cmake/Modules/CMakeCCEInformation.cmake | 41 + .../Modules/CMakeDetermineCCECompiler.cmake | 124 +++ .../cmake/Modules/CMakeTestCCECompiler.cmake | 1 + .../cmake/cpu/CMakeLists.txt | 37 + .../cmake/npu/CMakeLists.txt | 26 + .../AddKernelInvocation/data_utils.h | 203 +++++ .../AddKernelInvocation/main.cpp | 82 ++ .../AddKernelInvocation/run.sh | 130 +++ .../AddKernelInvocation/scripts/acl.json | 1 + .../AddKernelInvocation/scripts/gen_data.py | 27 + .../scripts/verify_result.py | 53 ++ .../CommonMistake/AclTensorFreeFail/README.md | 45 + .../AclTensorFreeFail/run_test.sh | 12 + .../AddressMisalign/AclNNInvocation/README.md | 54 ++ .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 180 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../AddressMisalign/AclNNInvocation/run.sh | 101 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 26 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 80 ++ .../AclNNInvocation/src/main.cpp | 162 ++++ .../AclNNInvocation/src/op_runner.cpp | 453 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 51 ++ .../AddressMisalign/AddCustom/CMakeLists.txt | 69 ++ .../AddCustom/CMakePresets.json | 63 ++ .../AddressMisalign/AddCustom/build.sh | 37 + .../AddCustom/cmake/config.cmake | 25 + .../AddCustom/cmake/func.cmake | 228 +++++ .../AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 73 ++ .../AddCustom/op_host/add_custom_tiling.h | 22 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 96 ++ .../AddCustom/scripts/install.sh | 318 +++++++ .../AddCustom/scripts/upgrade.sh | 151 ++++ .../AddKernelInvocation/CMakeLists.txt | 10 + .../AddKernelInvocation/README.md | 81 ++ .../AddKernelInvocation/add_custom.cpp | 91 ++ .../cmake/Modules/CMakeCCECompiler.cmake.in | 5 + .../cmake/Modules/CMakeCCEInformation.cmake | 41 + .../Modules/CMakeDetermineCCECompiler.cmake | 124 +++ .../cmake/Modules/CMakeTestCCECompiler.cmake | 1 + .../cmake/cpu/CMakeLists.txt | 37 + .../cmake/npu/CMakeLists.txt | 26 + .../AddKernelInvocation/data_utils.h | 203 +++++ .../AddKernelInvocation/main.cpp | 82 ++ .../AddKernelInvocation/run.sh | 130 +++ .../AddKernelInvocation/scripts/acl.json | 1 + .../AddKernelInvocation/scripts/gen_data.py | 27 + .../scripts/verify_result.py | 53 ++ .../CommonMistake/AddressMisalign/README.md | 52 ++ .../CommonMistake/AddressMisalign/run_test.sh | 12 + .../InvalidMemory/AclNNInvocation/README.md | 52 ++ .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../InvalidMemory/AclNNInvocation/run.sh | 99 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 16 + .../AclNNInvocation/scripts/verify_result.py | 23 + .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 163 ++++ .../AclNNInvocation/src/op_runner.cpp | 452 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../InvalidMemory/AddCustom/CMakeLists.txt | 69 ++ .../InvalidMemory/AddCustom/CMakePresets.json | 63 ++ .../InvalidMemory/AddCustom/build.sh | 37 + .../AddCustom/cmake/config.cmake | 25 + .../InvalidMemory/AddCustom/cmake/func.cmake | 228 +++++ .../InvalidMemory/AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 264 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 72 ++ .../AddCustom/op_host/add_custom_tiling.h | 16 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 83 ++ .../AddCustom/scripts/install.sh | 318 +++++++ .../AddCustom/scripts/upgrade.sh | 151 ++++ .../AddKernelInvocation/CMakeLists.txt | 10 + .../AddKernelInvocation/README.md | 81 ++ .../AddKernelInvocation/add_custom.cpp | 83 ++ .../cceprint/add_custom_0_0_mix.cce | 117 +++ .../cmake/Modules/CMakeCCECompiler.cmake.in | 5 + .../cmake/Modules/CMakeCCEInformation.cmake | 41 + .../Modules/CMakeDetermineCCECompiler.cmake | 124 +++ .../cmake/Modules/CMakeTestCCECompiler.cmake | 1 + .../cmake/cpu/CMakeLists.txt | 37 + .../cmake/npu/CMakeLists.txt | 26 + .../AddKernelInvocation/data_utils.h | 203 +++++ .../AddKernelInvocation/main.cpp | 82 ++ .../InvalidMemory/AddKernelInvocation/run.sh | 130 +++ .../AddKernelInvocation/scripts/acl.json | 1 + .../AddKernelInvocation/scripts/gen_data.py | 27 + .../scripts/verify_result.py | 53 ++ .../CommonMistake/InvalidMemory/README.md | 51 ++ .../AclNNInvocation/README.md | 54 ++ .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 180 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../AclNNInvocation/run.sh | 101 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 25 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 80 ++ .../AclNNInvocation/src/main.cpp | 162 ++++ .../AclNNInvocation/src/op_runner.cpp | 453 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 51 ++ .../AddCustom/CMakeLists.txt | 69 ++ .../AddCustom/CMakePresets.json | 63 ++ .../AddCustom/build.sh | 37 + .../AddCustom/cmake/config.cmake | 25 + .../AddCustom/cmake/func.cmake | 228 +++++ .../AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 73 ++ .../AddCustom/op_host/add_custom_tiling.h | 22 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 96 ++ .../AddCustom/scripts/install.sh | 318 +++++++ .../AddCustom/scripts/upgrade.sh | 151 ++++ .../AddKernelInvocation/CMakeLists.txt | 10 + .../AddKernelInvocation/README.md | 81 ++ .../AddKernelInvocation/add_custom.cpp | 91 ++ .../cceprint/add_custom_0_0_mix.cce | 15 + .../cmake/Modules/CMakeCCECompiler.cmake.in | 5 + .../cmake/Modules/CMakeCCEInformation.cmake | 41 + .../Modules/CMakeDetermineCCECompiler.cmake | 124 +++ .../cmake/Modules/CMakeTestCCECompiler.cmake | 1 + .../cmake/cpu/CMakeLists.txt | 37 + .../cmake/npu/CMakeLists.txt | 26 + .../AddKernelInvocation/data_utils.h | 203 +++++ .../AddKernelInvocation/main.cpp | 82 ++ .../AddKernelInvocation/run.sh | 130 +++ .../AddKernelInvocation/scripts/acl.json | 1 + .../AddKernelInvocation/scripts/gen_data.py | 27 + .../scripts/verify_result.py | 53 ++ .../MPUAddressAccessInvalid/README.md | 63 ++ .../NoTilingKey/AclNNInvocation/README.md | 52 ++ .../NoTilingKey/AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../NoTilingKey/AclNNInvocation/run.sh | 99 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 16 + .../AclNNInvocation/scripts/verify_result.py | 23 + .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../NoTilingKey/AclNNInvocation/src/main.cpp | 163 ++++ .../AclNNInvocation/src/op_runner.cpp | 452 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../NoTilingKey/AddCustom/CMakeLists.txt | 69 ++ .../NoTilingKey/AddCustom/CMakePresets.json | 63 ++ .../NoTilingKey/AddCustom/build.sh | 37 + .../NoTilingKey/AddCustom/cmake/config.cmake | 25 + .../NoTilingKey/AddCustom/cmake/func.cmake | 228 +++++ .../NoTilingKey/AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 264 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 72 ++ .../AddCustom/op_host/add_custom_tiling.h | 16 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 129 +++ .../NoTilingKey/AddCustom/scripts/install.sh | 318 +++++++ .../NoTilingKey/AddCustom/scripts/upgrade.sh | 151 ++++ .../CommonMistake/NoTilingKey/README.md | 38 + .../OpNameError/AclNNInvocation/README.md | 52 ++ .../OpNameError/AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../OpNameError/AclNNInvocation/run.sh | 99 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 16 + .../AclNNInvocation/scripts/verify_result.py | 23 + .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../OpNameError/AclNNInvocation/src/main.cpp | 163 ++++ .../AclNNInvocation/src/op_runner.cpp | 452 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../OpNameError/AddCustom/CMakeLists.txt | 69 ++ .../OpNameError/AddCustom/CMakePresets.json | 63 ++ .../OpNameError/AddCustom/build.sh | 80 ++ .../OpNameError/AddCustom/cmake/config.cmake | 25 + .../OpNameError/AddCustom/cmake/func.cmake | 192 ++++ .../OpNameError/AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 127 +++ .../cmake/util/ascendc_impl_build.py | 501 +++++++++++ .../cmake/util/ascendc_ops_config.py | 120 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 58 ++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 20 + .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 242 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 294 +++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 87 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tensorflow_add_custom_cc_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom_cc.cpp | 65 ++ .../AddCustom/op_host/add_custom_cc_tiling.h | 11 + .../AddCustom/op_kernel/CMakeLists.txt | 68 ++ .../AddCustom/op_kernel/add_custom1.cpp | 80 ++ .../OpNameError/AddCustom/scripts/install.sh | 318 +++++++ .../OpNameError/AddCustom/scripts/upgrade.sh | 151 ++++ .../CommonMistake/OpNameError/README.md | 40 + operator_contrib/CommonMistake/README.md | 16 + .../ResultError/AclNNInvocation/README.md | 54 ++ .../ResultError/AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 180 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../ResultError/AclNNInvocation/run.sh | 101 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 25 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 80 ++ .../ResultError/AclNNInvocation/src/main.cpp | 162 ++++ .../AclNNInvocation/src/op_runner.cpp | 453 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 51 ++ .../ResultError/AddCustom/CMakeLists.txt | 69 ++ .../ResultError/AddCustom/CMakePresets.json | 63 ++ .../ResultError/AddCustom/build.sh | 37 + .../ResultError/AddCustom/cmake/config.cmake | 25 + .../ResultError/AddCustom/cmake/func.cmake | 228 +++++ .../ResultError/AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 73 ++ .../AddCustom/op_host/add_custom_tiling.h | 22 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 93 ++ .../ResultError/AddCustom/scripts/install.sh | 318 +++++++ .../ResultError/AddCustom/scripts/upgrade.sh | 151 ++++ .../CommonMistake/ResultError/README.md | 38 + .../AclNNInvocation/README.md | 54 ++ .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 180 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../SegmentationFault/AclNNInvocation/run.sh | 101 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 26 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 80 ++ .../AclNNInvocation/src/main.cpp | 162 ++++ .../AclNNInvocation/src/op_runner.cpp | 453 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 51 ++ .../AddCustom/CMakeLists.txt | 69 ++ .../AddCustom/CMakePresets.json | 63 ++ .../SegmentationFault/AddCustom/build.sh | 37 + .../AddCustom/cmake/config.cmake | 25 + .../AddCustom/cmake/func.cmake | 228 +++++ .../AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 83 ++ .../AddCustom/op_host/add_custom_tiling.h | 23 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 96 ++ .../AddCustom/scripts/install.sh | 318 +++++++ .../AddCustom/scripts/upgrade.sh | 151 ++++ .../CommonMistake/SegmentationFault/README.md | 32 + .../SegmentationFault/run_test.sh | 12 + .../SegmentationFault/tiling_debug.py | 15 + .../SynchronizeFail/AclNNInvocation/README.md | 54 ++ .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 180 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../SynchronizeFail/AclNNInvocation/run.sh | 101 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 26 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 80 ++ .../AclNNInvocation/src/main.cpp | 162 ++++ .../AclNNInvocation/src/op_runner.cpp | 453 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 51 ++ .../SynchronizeFail/AddCustom/CMakeLists.txt | 69 ++ .../AddCustom/CMakePresets.json | 63 ++ .../SynchronizeFail/AddCustom/build.sh | 37 + .../AddCustom/cmake/config.cmake | 25 + .../AddCustom/cmake/func.cmake | 228 +++++ .../AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 73 ++ .../AddCustom/op_host/add_custom_tiling.h | 22 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 97 +++ .../AddCustom/scripts/install.sh | 318 +++++++ .../AddCustom/scripts/upgrade.sh | 151 ++++ .../AddKernelInvocation/CMakeLists.txt | 10 + .../AddKernelInvocation/README.md | 81 ++ .../AddKernelInvocation/add_custom.cpp | 91 ++ .../cmake/Modules/CMakeCCECompiler.cmake.in | 5 + .../cmake/Modules/CMakeCCEInformation.cmake | 41 + .../Modules/CMakeDetermineCCECompiler.cmake | 124 +++ .../cmake/Modules/CMakeTestCCECompiler.cmake | 1 + .../cmake/cpu/CMakeLists.txt | 37 + .../cmake/npu/CMakeLists.txt | 26 + .../AddKernelInvocation/data_utils.h | 203 +++++ .../AddKernelInvocation/main.cpp | 82 ++ .../AddKernelInvocation/run.sh | 130 +++ .../AddKernelInvocation/scripts/acl.json | 1 + .../AddKernelInvocation/scripts/gen_data.py | 27 + .../scripts/verify_result.py | 53 ++ .../CommonMistake/SynchronizeFail/README.md | 56 ++ .../CommonMistake/SynchronizeFail/run_test.sh | 12 + .../TimeOut/AclNNInvocation/README.md | 52 ++ .../TimeOut/AclNNInvocation/inc/common.h | 45 + .../TimeOut/AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../TimeOut/AclNNInvocation/run.sh | 116 +++ .../TimeOut/AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 16 + .../AclNNInvocation/scripts/verify_result.py | 23 + .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../TimeOut/AclNNInvocation/src/common.cpp | 79 ++ .../TimeOut/AclNNInvocation/src/main.cpp | 163 ++++ .../TimeOut/AclNNInvocation/src/op_runner.cpp | 452 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../TimeOut/AddCustom/CMakeLists.txt | 69 ++ .../TimeOut/AddCustom/CMakePresets.json | 63 ++ .../CommonMistake/TimeOut/AddCustom/build.sh | 37 + .../TimeOut/AddCustom/cmake/config.cmake | 25 + .../TimeOut/AddCustom/cmake/func.cmake | 228 +++++ .../TimeOut/AddCustom/cmake/intf.cmake | 26 + .../TimeOut/AddCustom/cmake/makeself.cmake | 17 + .../TimeOut/AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../TimeOut/AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 264 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../TimeOut/AddCustom/op_host/CMakeLists.txt | 82 ++ .../TimeOut/AddCustom/op_host/add_custom.cpp | 71 ++ .../AddCustom/op_host/add_custom_tiling.h | 16 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 99 +++ .../TimeOut/AddCustom/scripts/install.sh | 318 +++++++ .../TimeOut/AddCustom/scripts/upgrade.sh | 151 ++++ .../CommonMistake/TimeOut/README.md | 23 + .../AclNNInvocation/README.md | 54 ++ .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 180 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../AclNNInvocation/run.sh | 119 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 25 + .../AclNNInvocation/scripts/get_time.py | 31 + .../AclNNInvocation/scripts/verify_result.py | 40 + .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 80 ++ .../AclNNInvocation/src/main.cpp | 162 ++++ .../AclNNInvocation/src/op_runner.cpp | 453 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 51 ++ .../AddCustom/CMakeLists.txt | 69 ++ .../AddCustom/CMakePresets.json | 63 ++ .../TimeOverrunBaseline/AddCustom/build.sh | 37 + .../AddCustom/cmake/config.cmake | 25 + .../AddCustom/cmake/func.cmake | 228 +++++ .../AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 73 ++ .../AddCustom/op_host/add_custom_tiling.h | 22 + .../AddCustom/op_kernel/CMakeLists.txt | 62 ++ .../AddCustom/op_kernel/add_custom.cpp | 84 ++ .../AddCustom/scripts/install.sh | 318 +++++++ .../AddCustom/scripts/upgrade.sh | 151 ++++ .../TimeOverrunBaseline/README.md | 20 + .../TypeDeletion/AclNNInvocation/README.md | 54 ++ .../TypeDeletion/AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 180 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../TypeDeletion/AclNNInvocation/run.sh | 101 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 26 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 67 ++ .../AclNNInvocation/src/common.cpp | 80 ++ .../TypeDeletion/AclNNInvocation/src/main.cpp | 162 ++++ .../AclNNInvocation/src/op_runner.cpp | 453 ++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 51 ++ .../TypeDeletion/AddCustom/CMakeLists.txt | 69 ++ .../TypeDeletion/AddCustom/CMakePresets.json | 63 ++ .../TypeDeletion/AddCustom/build.sh | 37 + .../TypeDeletion/AddCustom/cmake/config.cmake | 25 + .../TypeDeletion/AddCustom/cmake/func.cmake | 228 +++++ .../TypeDeletion/AddCustom/cmake/intf.cmake | 26 + .../AddCustom/cmake/makeself.cmake | 17 + .../AddCustom/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 129 +++ .../cmake/util/ascendc_impl_build.py | 446 ++++++++++ .../cmake/util/ascendc_ops_config.py | 114 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 115 +++ .../AddCustom/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 57 ++ .../AddCustom/cmake/util/gen_ops_filter.sh | 62 ++ .../AddCustom/cmake/util/gen_version_info.sh | 6 + .../AddCustom/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 248 ++++++ .../AddCustom/cmake/util/kernel_entry.py | 115 +++ .../AddCustom/cmake/util/kernel_impl.temp | 10 + .../AddCustom/cmake/util/makeself/COPYING | 339 ++++++++ .../AddCustom/cmake/util/makeself/README.md | 246 ++++++ .../AddCustom/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../AddCustom/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../AddCustom/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../AddCustom/cmake/util/opdesc_parser.py | 260 ++++++ .../AddCustom/cmake/util/parse_ini_to_json.py | 338 +++++++ .../AddCustom/cmake/util/preset_parse.py | 23 + .../AddCustom/cmake/util/replay_codegen.py | 105 +++ .../AddCustom/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 84 ++ .../AddCustom/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_add_custom_plugin.cc | 23 + .../AddCustom/op_host/CMakeLists.txt | 82 ++ .../AddCustom/op_host/add_custom.cpp | 73 ++ .../AddCustom/op_host/add_custom_tiling.h | 22 + .../AddCustom/op_kernel/CMakeLists.txt | 61 ++ .../AddCustom/op_kernel/add_custom.cpp | 96 ++ .../TypeDeletion/AddCustom/scripts/install.sh | 318 +++++++ .../TypeDeletion/AddCustom/scripts/upgrade.sh | 151 ++++ .../CommonMistake/TypeDeletion/README.md | 34 + .../CommonMistake/TypeDeletion/run_test.sh | 4 + .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 60 ++ .../DifferentAlgorithm/AclNNInvocation/run.sh | 109 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 24 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 68 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 163 ++++ .../AclNNInvocation/src/op_runner.cpp | 485 +++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../DifferentAlgorithm/Demo.json | 22 + .../DifferentAlgorithm/Demo/CMakeLists.txt | 74 ++ .../DifferentAlgorithm/Demo/CMakePresets.json | 63 ++ .../DifferentAlgorithm/Demo/build.sh | 95 ++ .../Demo/cmake/config.cmake | 30 + .../DifferentAlgorithm/Demo/cmake/func.cmake | 212 +++++ .../DifferentAlgorithm/Demo/cmake/intf.cmake | 28 + .../Demo/cmake/makeself.cmake | 33 + .../Demo/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 498 +++++++++++ .../Demo/cmake/util/ascendc_gen_options.py | 86 ++ .../Demo/cmake/util/ascendc_impl_build.py | 657 ++++++++++++++ .../Demo/cmake/util/ascendc_ops_config.py | 291 +++++++ .../Demo/cmake/util/ascendc_replay_build.py | 65 ++ .../Demo/cmake/util/batch_replay_impl.temp | 117 +++ .../Demo/cmake/util/code_channel_infer.py | 58 ++ .../Demo/cmake/util/const_var.py | 55 ++ .../cmake/util/gen_impl_and_mrege_json.sh | 21 + .../Demo/cmake/util/gen_ops_filter.sh | 62 ++ .../Demo/cmake/util/gen_version_info.sh | 10 + .../Demo/cmake/util/insert_op_info.py | 36 + .../Demo/cmake/util/insert_simplified_keys.py | 247 ++++++ .../Demo/cmake/util/kernel_entry.py | 115 +++ .../Demo/cmake/util/kernel_impl.temp | 10 + .../Demo/cmake/util/makeself/COPYING | 339 ++++++++ .../Demo/cmake/util/makeself/README.md | 246 ++++++ .../Demo/cmake/util/makeself/VERSION | 1 + .../Demo/cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../Demo/cmake/util/makeself/makeself.1 | 110 +++ .../Demo/cmake/util/makeself/makeself.lsm | 16 + .../Demo/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../Demo/cmake/util/makeself/run-tests.sh | 8 + .../Demo/cmake/util/merge_aicpu_info_json.sh | 32 + .../Demo/cmake/util/opdesc_parser.py | 346 ++++++++ .../Demo/cmake/util/parse_ini_to_json.py | 346 ++++++++ .../Demo/cmake/util/preset_parse.py | 35 + .../Demo/cmake/util/replay_codegen.py | 105 +++ .../Demo/cmake/util/replay_impl.temp | 120 +++ .../Demo/cmake/util/tiling_data_def_build.py | 88 ++ .../Demo/framework/CMakeLists.txt | 11 + .../Demo/framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_demo_plugin.cc | 23 + .../Demo/op_host/CMakeLists.txt | 93 ++ .../DifferentAlgorithm/Demo/op_host/demo.cpp | 61 ++ .../Demo/op_host/demo_tiling.h | 10 + .../Demo/op_kernel/CMakeLists.txt | 68 ++ .../Demo/op_kernel/demo.cpp | 57 ++ .../Demo/scripts/install.sh | 318 +++++++ .../Demo/scripts/upgrade.sh | 145 +++ .../DifferentAlgorithm/DemoFix/CMakeLists.txt | 74 ++ .../DemoFix/CMakePresets.json | 63 ++ .../DifferentAlgorithm/DemoFix/build.sh | 95 ++ .../DemoFix/cmake/config.cmake | 30 + .../DemoFix/cmake/func.cmake | 212 +++++ .../DemoFix/cmake/intf.cmake | 28 + .../DemoFix/cmake/makeself.cmake | 33 + .../DemoFix/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 498 +++++++++++ .../DemoFix/cmake/util/ascendc_gen_options.py | 86 ++ .../DemoFix/cmake/util/ascendc_impl_build.py | 657 ++++++++++++++ .../DemoFix/cmake/util/ascendc_ops_config.py | 291 +++++++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../DemoFix/cmake/util/batch_replay_impl.temp | 117 +++ .../DemoFix/cmake/util/code_channel_infer.py | 58 ++ .../DemoFix/cmake/util/const_var.py | 55 ++ .../cmake/util/gen_impl_and_mrege_json.sh | 21 + .../DemoFix/cmake/util/gen_ops_filter.sh | 62 ++ .../DemoFix/cmake/util/gen_version_info.sh | 10 + .../DemoFix/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 247 ++++++ .../DemoFix/cmake/util/kernel_entry.py | 115 +++ .../DemoFix/cmake/util/kernel_impl.temp | 10 + .../DemoFix/cmake/util/makeself/COPYING | 339 ++++++++ .../DemoFix/cmake/util/makeself/README.md | 246 ++++++ .../DemoFix/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../DemoFix/cmake/util/makeself/makeself.1 | 110 +++ .../DemoFix/cmake/util/makeself/makeself.lsm | 16 + .../DemoFix/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../DemoFix/cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 32 + .../DemoFix/cmake/util/opdesc_parser.py | 346 ++++++++ .../DemoFix/cmake/util/parse_ini_to_json.py | 346 ++++++++ .../DemoFix/cmake/util/preset_parse.py | 35 + .../DemoFix/cmake/util/replay_codegen.py | 105 +++ .../DemoFix/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 88 ++ .../DemoFix/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_demo_plugin.cc | 23 + .../DemoFix/op_host/CMakeLists.txt | 93 ++ .../DemoFix/op_host/demo.cpp | 61 ++ .../DemoFix/op_host/demo_tiling.h | 10 + .../DemoFix/op_kernel/CMakeLists.txt | 68 ++ .../DemoFix/op_kernel/demo.cpp | 58 ++ .../DemoFix/scripts/install.sh | 318 +++++++ .../DemoFix/scripts/upgrade.sh | 145 +++ .../DifferentAlgorithm/README.md | 162 ++++ .../DifferentAlgorithm/run_test.sh | 20 + operator_contrib/PrecisionTips/README.md | 7 + .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 60 ++ .../AclNNInvocation/run.sh | 109 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 28 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 68 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 165 ++++ .../AclNNInvocation/src/op_runner.cpp | 485 +++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../SequenceOperationSteps/Demo.json | 28 + .../Demo/CMakeLists.txt | 74 ++ .../Demo/CMakePresets.json | 63 ++ .../SequenceOperationSteps/Demo/build.sh | 95 ++ .../Demo/cmake/config.cmake | 30 + .../Demo/cmake/func.cmake | 212 +++++ .../Demo/cmake/intf.cmake | 28 + .../Demo/cmake/makeself.cmake | 33 + .../Demo/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 498 +++++++++++ .../Demo/cmake/util/ascendc_gen_options.py | 86 ++ .../Demo/cmake/util/ascendc_impl_build.py | 657 ++++++++++++++ .../Demo/cmake/util/ascendc_ops_config.py | 291 +++++++ .../Demo/cmake/util/ascendc_replay_build.py | 65 ++ .../Demo/cmake/util/batch_replay_impl.temp | 117 +++ .../Demo/cmake/util/code_channel_infer.py | 58 ++ .../Demo/cmake/util/const_var.py | 55 ++ .../cmake/util/gen_impl_and_mrege_json.sh | 21 + .../Demo/cmake/util/gen_ops_filter.sh | 62 ++ .../Demo/cmake/util/gen_version_info.sh | 10 + .../Demo/cmake/util/insert_op_info.py | 36 + .../Demo/cmake/util/insert_simplified_keys.py | 247 ++++++ .../Demo/cmake/util/kernel_entry.py | 115 +++ .../Demo/cmake/util/kernel_impl.temp | 10 + .../Demo/cmake/util/makeself/COPYING | 339 ++++++++ .../Demo/cmake/util/makeself/README.md | 246 ++++++ .../Demo/cmake/util/makeself/VERSION | 1 + .../Demo/cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../Demo/cmake/util/makeself/makeself.1 | 110 +++ .../Demo/cmake/util/makeself/makeself.lsm | 16 + .../Demo/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../Demo/cmake/util/makeself/run-tests.sh | 8 + .../Demo/cmake/util/merge_aicpu_info_json.sh | 32 + .../Demo/cmake/util/opdesc_parser.py | 346 ++++++++ .../Demo/cmake/util/parse_ini_to_json.py | 346 ++++++++ .../Demo/cmake/util/preset_parse.py | 35 + .../Demo/cmake/util/replay_codegen.py | 105 +++ .../Demo/cmake/util/replay_impl.temp | 120 +++ .../Demo/cmake/util/tiling_data_def_build.py | 88 ++ .../Demo/framework/CMakeLists.txt | 11 + .../Demo/framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_demo_plugin.cc | 23 + .../Demo/op_host/CMakeLists.txt | 93 ++ .../Demo/op_host/demo.cpp | 66 ++ .../Demo/op_host/demo_tiling.h | 10 + .../Demo/op_kernel/CMakeLists.txt | 68 ++ .../Demo/op_kernel/demo.cpp | 66 ++ .../Demo/scripts/install.sh | 318 +++++++ .../Demo/scripts/upgrade.sh | 145 +++ .../DemoFix/CMakeLists.txt | 74 ++ .../DemoFix/CMakePresets.json | 63 ++ .../SequenceOperationSteps/DemoFix/build.sh | 95 ++ .../DemoFix/cmake/config.cmake | 30 + .../DemoFix/cmake/func.cmake | 212 +++++ .../DemoFix/cmake/intf.cmake | 28 + .../DemoFix/cmake/makeself.cmake | 33 + .../DemoFix/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 498 +++++++++++ .../DemoFix/cmake/util/ascendc_gen_options.py | 86 ++ .../DemoFix/cmake/util/ascendc_impl_build.py | 657 ++++++++++++++ .../DemoFix/cmake/util/ascendc_ops_config.py | 291 +++++++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../DemoFix/cmake/util/batch_replay_impl.temp | 117 +++ .../DemoFix/cmake/util/code_channel_infer.py | 58 ++ .../DemoFix/cmake/util/const_var.py | 55 ++ .../cmake/util/gen_impl_and_mrege_json.sh | 21 + .../DemoFix/cmake/util/gen_ops_filter.sh | 62 ++ .../DemoFix/cmake/util/gen_version_info.sh | 10 + .../DemoFix/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 247 ++++++ .../DemoFix/cmake/util/kernel_entry.py | 115 +++ .../DemoFix/cmake/util/kernel_impl.temp | 10 + .../DemoFix/cmake/util/makeself/COPYING | 339 ++++++++ .../DemoFix/cmake/util/makeself/README.md | 246 ++++++ .../DemoFix/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../DemoFix/cmake/util/makeself/makeself.1 | 110 +++ .../DemoFix/cmake/util/makeself/makeself.lsm | 16 + .../DemoFix/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../DemoFix/cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 32 + .../DemoFix/cmake/util/opdesc_parser.py | 346 ++++++++ .../DemoFix/cmake/util/parse_ini_to_json.py | 346 ++++++++ .../DemoFix/cmake/util/preset_parse.py | 35 + .../DemoFix/cmake/util/replay_codegen.py | 105 +++ .../DemoFix/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 88 ++ .../DemoFix/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_demo_plugin.cc | 23 + .../DemoFix/op_host/CMakeLists.txt | 93 ++ .../DemoFix/op_host/demo.cpp | 66 ++ .../DemoFix/op_host/demo_tiling.h | 10 + .../DemoFix/op_kernel/CMakeLists.txt | 68 ++ .../DemoFix/op_kernel/demo.cpp | 66 ++ .../DemoFix/scripts/install.sh | 318 +++++++ .../DemoFix/scripts/upgrade.sh | 145 +++ .../SequenceOperationSteps/README.md | 157 ++++ .../SequenceOperationSteps/run_test.sh | 20 + .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 60 ++ .../TypeConversion/AclNNInvocation/run.sh | 109 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 27 + .../AclNNInvocation/scripts/verify_result.py | 53 ++ .../AclNNInvocation/src/CMakeLists.txt | 68 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 163 ++++ .../AclNNInvocation/src/op_runner.cpp | 485 +++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../PrecisionTips/TypeConversion/Demo.json | 22 + .../TypeConversion/Demo/CMakeLists.txt | 74 ++ .../TypeConversion/Demo/CMakePresets.json | 63 ++ .../TypeConversion/Demo/build.sh | 95 ++ .../TypeConversion/Demo/cmake/config.cmake | 30 + .../TypeConversion/Demo/cmake/func.cmake | 212 +++++ .../TypeConversion/Demo/cmake/intf.cmake | 28 + .../TypeConversion/Demo/cmake/makeself.cmake | 33 + .../Demo/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 498 +++++++++++ .../Demo/cmake/util/ascendc_gen_options.py | 86 ++ .../Demo/cmake/util/ascendc_impl_build.py | 657 ++++++++++++++ .../Demo/cmake/util/ascendc_ops_config.py | 291 +++++++ .../Demo/cmake/util/ascendc_replay_build.py | 65 ++ .../Demo/cmake/util/batch_replay_impl.temp | 117 +++ .../Demo/cmake/util/code_channel_infer.py | 58 ++ .../Demo/cmake/util/const_var.py | 55 ++ .../cmake/util/gen_impl_and_mrege_json.sh | 21 + .../Demo/cmake/util/gen_ops_filter.sh | 62 ++ .../Demo/cmake/util/gen_version_info.sh | 10 + .../Demo/cmake/util/insert_op_info.py | 36 + .../Demo/cmake/util/insert_simplified_keys.py | 247 ++++++ .../Demo/cmake/util/kernel_entry.py | 115 +++ .../Demo/cmake/util/kernel_impl.temp | 10 + .../Demo/cmake/util/makeself/COPYING | 339 ++++++++ .../Demo/cmake/util/makeself/README.md | 246 ++++++ .../Demo/cmake/util/makeself/VERSION | 1 + .../Demo/cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../Demo/cmake/util/makeself/makeself.1 | 110 +++ .../Demo/cmake/util/makeself/makeself.lsm | 16 + .../Demo/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../Demo/cmake/util/makeself/run-tests.sh | 8 + .../Demo/cmake/util/merge_aicpu_info_json.sh | 32 + .../Demo/cmake/util/opdesc_parser.py | 346 ++++++++ .../Demo/cmake/util/parse_ini_to_json.py | 346 ++++++++ .../Demo/cmake/util/preset_parse.py | 35 + .../Demo/cmake/util/replay_codegen.py | 105 +++ .../Demo/cmake/util/replay_impl.temp | 120 +++ .../Demo/cmake/util/tiling_data_def_build.py | 88 ++ .../Demo/framework/CMakeLists.txt | 11 + .../Demo/framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_demo_plugin.cc | 23 + .../Demo/op_host/CMakeLists.txt | 93 ++ .../TypeConversion/Demo/op_host/demo.cpp | 61 ++ .../TypeConversion/Demo/op_host/demo_tiling.h | 10 + .../Demo/op_kernel/CMakeLists.txt | 68 ++ .../TypeConversion/Demo/op_kernel/demo.cpp | 67 ++ .../TypeConversion/Demo/scripts/install.sh | 318 +++++++ .../TypeConversion/Demo/scripts/upgrade.sh | 145 +++ .../TypeConversion/DemoFix/CMakeLists.txt | 74 ++ .../TypeConversion/DemoFix/CMakePresets.json | 63 ++ .../TypeConversion/DemoFix/build.sh | 95 ++ .../TypeConversion/DemoFix/cmake/config.cmake | 30 + .../TypeConversion/DemoFix/cmake/func.cmake | 212 +++++ .../TypeConversion/DemoFix/cmake/intf.cmake | 28 + .../DemoFix/cmake/makeself.cmake | 33 + .../DemoFix/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 498 +++++++++++ .../DemoFix/cmake/util/ascendc_gen_options.py | 86 ++ .../DemoFix/cmake/util/ascendc_impl_build.py | 657 ++++++++++++++ .../DemoFix/cmake/util/ascendc_ops_config.py | 291 +++++++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../DemoFix/cmake/util/batch_replay_impl.temp | 117 +++ .../DemoFix/cmake/util/code_channel_infer.py | 58 ++ .../DemoFix/cmake/util/const_var.py | 55 ++ .../cmake/util/gen_impl_and_mrege_json.sh | 21 + .../DemoFix/cmake/util/gen_ops_filter.sh | 62 ++ .../DemoFix/cmake/util/gen_version_info.sh | 10 + .../DemoFix/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 247 ++++++ .../DemoFix/cmake/util/kernel_entry.py | 115 +++ .../DemoFix/cmake/util/kernel_impl.temp | 10 + .../DemoFix/cmake/util/makeself/COPYING | 339 ++++++++ .../DemoFix/cmake/util/makeself/README.md | 246 ++++++ .../DemoFix/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../DemoFix/cmake/util/makeself/makeself.1 | 110 +++ .../DemoFix/cmake/util/makeself/makeself.lsm | 16 + .../DemoFix/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../DemoFix/cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 32 + .../DemoFix/cmake/util/opdesc_parser.py | 346 ++++++++ .../DemoFix/cmake/util/parse_ini_to_json.py | 346 ++++++++ .../DemoFix/cmake/util/preset_parse.py | 35 + .../DemoFix/cmake/util/replay_codegen.py | 105 +++ .../DemoFix/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 88 ++ .../DemoFix/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_demo_plugin.cc | 23 + .../DemoFix/op_host/CMakeLists.txt | 93 ++ .../TypeConversion/DemoFix/op_host/demo.cpp | 61 ++ .../DemoFix/op_host/demo_tiling.h | 10 + .../DemoFix/op_kernel/CMakeLists.txt | 68 ++ .../TypeConversion/DemoFix/op_kernel/demo.cpp | 66 ++ .../TypeConversion/DemoFix/scripts/install.sh | 318 +++++++ .../TypeConversion/DemoFix/scripts/upgrade.sh | 145 +++ .../PrecisionTips/TypeConversion/README.md | 173 ++++ .../PrecisionTips/TypeConversion/run_test.sh | 20 + 1168 files changed, 140083 insertions(+) create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/README.md create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/data_utils.h create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/AclTensorFreeFail/README.md create mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/README.md create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/README.md create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake create mode 100755 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/cpu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/npu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/data_utils.h create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/AddressMisalign/README.md create mode 100755 operator_contrib/CommonMistake/AddressMisalign/run_test.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/README.md create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/common.h create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/op_runner.h create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/operator_desc.h create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/run.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/acl.json create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/gen_data.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/verify_result.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/main.cpp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/op_runner.cpp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/build.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/README.md create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake create mode 100755 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/cpu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/npu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/data_utils.h create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/InvalidMemory/README.md create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/README.md create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/README.md create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake create mode 100755 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/cpu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/npu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/data_utils.h create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/README.md create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/README.md create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/common.h create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/op_runner.h create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/operator_desc.h create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/run.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/acl.json create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/gen_data.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/verify_result.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/main.cpp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/op_runner.cpp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/build.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/NoTilingKey/README.md create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/README.md create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/common.h create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/op_runner.h create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/operator_desc.h create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/run.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/acl.json create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/gen_data.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/verify_result.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/main.cpp create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/op_runner.cpp create mode 100755 operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/build.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/framework/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/OpNameError/README.md create mode 100644 operator_contrib/CommonMistake/README.md create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/README.md create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/ResultError/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/ResultError/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/ResultError/README.md create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/README.md create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/SegmentationFault/README.md create mode 100755 operator_contrib/CommonMistake/SegmentationFault/run_test.sh create mode 100644 operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/README.md create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/README.md create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/data_utils.h create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/SynchronizeFail/README.md create mode 100755 operator_contrib/CommonMistake/SynchronizeFail/run_test.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/README.md create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/common.h create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/op_runner.h create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/operator_desc.h create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/run.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/acl.json create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/gen_data.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/verify_result.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/main.cpp create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/op_runner.cpp create mode 100755 operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/build.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/framework/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/CMakeLists.txt create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/TimeOut/README.md create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/README.md create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/TimeOverrunBaseline/README.md create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/README.md create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/run.sh create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/operator_desc.cpp create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakePresets.json create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/build.sh create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/config.cmake create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/func.cmake create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/intf.cmake create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/makeself.cmake create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/__init__.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_replay_build.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/const_var.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_op_info.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_entry.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_impl.temp create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/COPYING create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/README.md create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/VERSION create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.1 create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/preset_parse.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_codegen.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_impl.temp create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/tiling_data_def_build.py create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/install.sh create mode 100755 operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/upgrade.sh create mode 100644 operator_contrib/CommonMistake/TypeDeletion/README.md create mode 100755 operator_contrib/CommonMistake/TypeDeletion/run_test.sh create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/run.sh create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/operator_desc.cpp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo.json create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakePresets.json create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/build.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/config.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/func.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/intf.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/makeself.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/__init__.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_gen_options.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/const_var.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_op_info.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_entry.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/COPYING create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/README.md create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/VERSION create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/preset_parse.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_codegen.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/install.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/upgrade.sh create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakePresets.json create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/build.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/config.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/func.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/intf.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/makeself.cmake create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/__init__.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_gen_options.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/const_var.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_op_info.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_entry.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/COPYING create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/README.md create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/VERSION create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/preset_parse.py create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_codegen.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/install.sh create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/upgrade.sh create mode 100644 operator_contrib/PrecisionTips/DifferentAlgorithm/README.md create mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh create mode 100644 operator_contrib/PrecisionTips/README.md create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/run.sh create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/operator_desc.cpp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo.json create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakePresets.json create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/build.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/config.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/func.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/intf.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/makeself.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/__init__.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_gen_options.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/const_var.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_op_info.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_entry.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/COPYING create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/README.md create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/VERSION create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/preset_parse.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_codegen.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/install.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/upgrade.sh create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakePresets.json create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/build.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/config.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/func.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/intf.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/makeself.cmake create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/__init__.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_gen_options.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/const_var.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_op_info.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_entry.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/COPYING create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/README.md create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/VERSION create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/preset_parse.py create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_codegen.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/install.sh create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/upgrade.sh create mode 100644 operator_contrib/PrecisionTips/SequenceOperationSteps/README.md create mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/run.sh create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/operator_desc.cpp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo.json create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/CMakePresets.json create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/build.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/config.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/func.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/intf.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/makeself.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/__init__.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_gen_options.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/const_var.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_op_info.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_entry.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/COPYING create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/README.md create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/VERSION create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/preset_parse.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_codegen.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/framework/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/install.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/upgrade.sh create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakePresets.json create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/build.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/config.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/func.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/intf.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/makeself.cmake create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/__init__.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_bin_param_build.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_gen_options.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_impl_build.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_ops_config.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/batch_replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/code_channel_infer.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/const_var.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_impl_and_mrege_json.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_ops_filter.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_version_info.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_op_info.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_simplified_keys.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_entry.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/COPYING create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/README.md create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/VERSION create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/make-release.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself-header.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.lsm create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/run-tests.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/merge_aicpu_info_json.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/opdesc_parser.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/parse_ini_to_json.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/preset_parse.py create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_codegen.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_impl.temp create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/install.sh create mode 100755 operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/upgrade.sh create mode 100644 operator_contrib/PrecisionTips/TypeConversion/README.md create mode 100755 operator_contrib/PrecisionTips/TypeConversion/run_test.sh diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..17607353d --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,60 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + bool keepDims; + bool ignoreNan; + char *dtype; + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/run.sh new file mode 100644 index 000000000..2ad176126 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/run.sh @@ -0,0 +1,116 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + # rm ./input/*.bin + rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret=`python3 scripts/verify_result.py output/output_z.bin output/golden.bin` + echo $ret + if [ "x$ret" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..92b5a5bcf --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..ddb6e4a31 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 0: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..ee73e930f --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include + ${BUILT_PKG_PATH}/include/aclnnop +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..5d5f2f5d2 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/main.cpp @@ -0,0 +1,164 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..0ab4e9914 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,485 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_add_custom.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto ret = aclDestroyTensor(inputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(inputBuffers_[i]); + + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devInputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostInputs_[i]); + } else { + ret = aclrtFreeHost(hostInputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto ret = aclDestroyTensor(outputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(outputBuffers_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devOutputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostOutputs_[i]); + } else { + ret = aclrtFreeHost(hostOutputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + + auto ret = aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute Operator success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/build.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..6fc77c881 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,73 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 1; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..323f3076f --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,22 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..0d7ed7244 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,96 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2025. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength ); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * this->tileLength + 128], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, + uint8_t *workspace, uint8_t *tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/CMakeLists.txt new file mode 100644 index 000000000..fd87c7620 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.16) +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_CXX_STANDARD 17) + +set(CCE_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules) +list(APPEND CMAKE_MODULE_PATH ${CCE_CMAKE_PATH}) +project(kernel_samples LANGUAGES CCE CXX) + +add_subdirectory(cmake/cpu) +add_subdirectory(cmake/npu) \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/README.md b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/README.md new file mode 100644 index 000000000..49bee5787 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/README.md @@ -0,0 +1,81 @@ +## 目录结构介绍 +``` +├── AddKernelInvocation +│ ├── cmake // 编译工程文件 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 验证输出数据和真值数据是否一致的验证脚本 +│ │ └── verify_result.py // 真值对比文件 +│ │── add_custom.cpp // 算子kernel实现 +│ │── CMakeLists.txt // 编译工程文件 +│ │── data_utils.h // 数据读入写出函数 +│ │── main.cpp // 主函数，调用算子的应用程序，含CPU域及NPU域调用 +│ └── run.sh // 编译运行算子的脚本 +``` +## 代码实现介绍 +本调用样例中实现的是固定shape为8*2048的Add算子。 +- kernel实现 + Add算子的数学表达式为： + ``` + z = x + y + ``` + 计算逻辑是：Ascend C提供的矢量计算接口的操作元素都为LocalTensor，输入数据需要先搬运进片上存储，然后使用计算接口完成两个输入参数相加，得到最终结果，再搬出到外部存储上。 + + Add算子的实现流程分为3个基本任务：CopyIn，Compute，CopyOut。CopyIn任务负责将Global Memory上的输入Tensor xGm和yGm搬运到Local Memory，分别存储在xLocal、yLocal，Compute任务负责对xLocal、yLocal执行加法操作，计算结果存储在zLocal中，CopyOut任务负责将输出数据从zLocal搬运至Global Memory上的输出Tensor zGm中。具体请参考[add_custom.cpp](./add_custom.cpp)。 + +- 调用实现 + 1. CPU侧运行验证主要通过ICPU_RUN_KF CPU调测宏等CPU调测库提供的接口来完成； + 2. NPU侧运行验证主要通过使用<<<>>>内核调用符来完成。 + + 应用程序通过ASCENDC_CPU_DEBUG 宏区分代码逻辑运行于CPU侧还是NPU侧。 + +## 运行样例算子 + - 打开样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/KernelLaunch/AddKernelInvocation + ``` + - 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ```` + + 配置仿真模式日志文件目录，默认为sim_log。 + ```bash + export CAMODEL_LOG_PATH=./sim_log + ``` + + - 样例执行 + + ```bash + bash run.sh -r [RUN_MODE] -v [SOC_VERSION] + ``` + - RUN_MODE：编译方式，可选择CPU调试，NPU仿真，NPU上板。支持参数为[cpu / sim / npu]，默认值为cpu。 + - SOC_VERSION：昇腾AI处理器型号，如果无法确定具体的[SOC_VERSION]，则在安装昇腾AI处理器的服务器执行npu-smi info命令进行查询，在查询到的“Name”前增加Ascend信息，例如“Name”对应取值为xxxyy，实际配置的[SOC_VERSION]值为Ascendxxxyy。支持以下参数取值（xxx请替换为具体取值）： + - Atlas 推理系列产品（Ascend 310P处理器）参数值：Ascend310P1、Ascend310P3 + - Atlas 训练系列产品参数值：AscendxxxA、AscendxxxB + - Atlas A2训练系列产品参数值：AscendxxxB1、AscendxxxB2、AscendxxxB3、AscendxxxB4 + + 注：针对Atlas 训练系列产品使用NPU仿真调试，会存在精度问题，可选择其他芯片进行NPU仿真调试。 + + 示例如下。 + ```bash + bash run.sh -r cpu -v Ascend310P1 + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2023/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp new file mode 100644 index 000000000..daf334f89 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp @@ -0,0 +1,91 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; + +constexpr int32_t TOTAL_LENGTH = 8 * 2048; // total length of data +constexpr int32_t USE_CORE_NUM = 1; // num of core used +constexpr int32_t BLOCK_LENGTH = TOTAL_LENGTH / USE_CORE_NUM; // length computed of each core +constexpr int32_t TILE_NUM = 8; // split data into 8 tiles for each core +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue +constexpr int32_t TILE_LENGTH = BLOCK_LENGTH / TILE_NUM / BUFFER_NUM; // seperate to 2 parts, due to double buffer + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z) + { + xGm.SetGlobalBuffer((__gm__ half *)x + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + yGm.SetGlobalBuffer((__gm__ half *)y + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + zGm.SetGlobalBuffer((__gm__ half *)z + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + pipe.InitBuffer(inQueueX, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + } + __aicore__ inline void Process() + { + int32_t loopCount = TILE_NUM * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * TILE_LENGTH], TILE_LENGTH); + DataCopy(yLocal, yGm[progress * TILE_LENGTH], TILE_LENGTH); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, TILE_LENGTH); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * TILE_LENGTH+128], zLocal, TILE_LENGTH); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +{ + KernelAdd op; + op.Init(x, y, z); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +{ + add_custom<<>>(x, y, z); +} +#endif diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in new file mode 100644 index 000000000..a9b5688ff --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in @@ -0,0 +1,5 @@ +set(CMAKE_CCE_COMPILER "@CMAKE_CCE_COMPILER@") +set(CMAKE_CCE_COMPILER_LOADED 1) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS @CMAKE_CCE_SOURCE_FILE_EXTENSIONS@) +set(CMAKE_CCE_OUTPUT_EXTENSION @CMAKE_CCE_OUTPUT_EXTENSION@) +set(CMAKE_CCE_COMPILER_ENV_VAR "@CMAKE_CCE_COMPILER_ENV_VAR@") diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake new file mode 100644 index 000000000..7d2fc2ddd --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake @@ -0,0 +1,41 @@ +include(CMakeCommonLanguageInclude) + +set(CMAKE_INCLUDE_FLAG_CCE "-I") + +if(UNIX) + set(CMAKE_CCE_OUTPUT_EXTENSION .o) +else() + set(CMAKE_CCE_OUTPUT_EXTENSION .obj) +endif() + +set(_INCLUDED_FILE 0) +set(CMAKE_SHARED_LIBRARY_CCE_FLAGS -fPIC) +set(CMAKE_SHARED_LIBRARY_CREATE_CCE_FLAGS -shared) +set(CMAKE_STATIC_LIBRARY_CREATE_CCE_FLAGS "--cce-build-static-lib") +set(CMAKE_LIBRARY_CREATE_CCE_FLAGS "--cce-fatobj-link") + +if(NOT CMAKE_CCE_COMPILE_OBJECT) + set(CMAKE_CCE_COMPILE_OBJECT + " -xcce ${__IMPLICIT_INCLUDES} ${_CMAKE_CCE_BUILTIN_INCLUDE_PATH} ${_CMAKE_COMPILE_AS_CCE_FLAG} ${_CMAKE_CCE_COMPILE_OPTIONS} ${_CMAKE_CCE_COMMON_COMPILE_OPTIONS} -pthread -o -c ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_LIBRARY) + set(CMAKE_CCE_CREATE_SHARED_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_STATIC_LIBRARY) + set(CMAKE_CCE_CREATE_STATIC_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_MODULE) + set(CMAKE_CCE_CREATE_SHARED_MODULE ${CMAKE_CCE_CREATE_SHARED_LIBRARY}) +endif() + +if(NOT CMAKE_CCE_LINK_EXECUTABLE) + set(CMAKE_CCE_LINK_EXECUTABLE + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ${__IMPLICIT_LINKS}") +endif() + +set(CMAKE_CCE_INFORMATION_LOADED 1) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake new file mode 100755 index 000000000..f3c0a4732 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake @@ -0,0 +1,124 @@ +find_program(CMAKE_CCE_COMPILER NAMES "ccec" PATHS "$ENV{PATH}" DOC "CCE Compiler") + +mark_as_advanced(CMAKE_CCE_COMPILER) + +message(STATUS "CMAKE_CCE_COMPILER: " ${CMAKE_CCE_COMPILER}) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS cce;cpp) +set(CMAKE_CCE_COMPILER_ENV_VAR "CCE") +message(STATUS "CMAKE_CURRENT_LIST_DIR: " ${CMAKE_CURRENT_LIST_DIR}) + +# configure all variables set in this file +configure_file(${CMAKE_CURRENT_LIST_DIR}/CMakeCCECompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeCCECompiler.cmake + @ONLY +) + +message(STATUS "ASCEND_PRODUCT_TYPE:\n" " ${ASCEND_PRODUCT_TYPE}") +message(STATUS "ASCEND_CORE_TYPE:\n" " ${ASCEND_CORE_TYPE}") +message(STATUS "ASCEND_INSTALL_PATH:\n" " ${ASCEND_INSTALL_PATH}") + +if(DEFINED ASCEND_INSTALL_PATH) + set(_CMAKE_ASCEND_INSTALL_PATH ${ASCEND_INSTALL_PATH}) +else() + message(FATAL_ERROR + "no, installation path found, should passing -DASCEND_INSTALL_PATH= in cmake" + ) + set(_CMAKE_ASCEND_INSTALL_PATH) +endif() + + +if(DEFINED ASCEND_PRODUCT_TYPE) + set(_CMAKE_CCE_COMMON_COMPILE_OPTIONS "--cce-auto-sync -mllvm -api-deps-filter") + if(ASCEND_PRODUCT_TYPE STREQUAL "") + message(FATAL_ERROR "ASCEND_PRODUCT_TYPE must be non-empty if set.") + elseif(ASCEND_PRODUCT_TYPE AND NOT ASCEND_PRODUCT_TYPE MATCHES "^Ascend[0-9][0-9][0-9][a-zA-Z]?[1-9]?$") + message(FATAL_ERROR + "ASCEND_PRODUCT_TYPE: ${ASCEND_PRODUCT_TYPE}\n" + "is not one of the following: Ascend910A, Ascend310P1, Ascend910B1, Ascend310B1" + ) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c100") + else() + message(FATAL_ERROR, "only AiCore inside") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310P1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-cube") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-record-overflow=false -mllvm -cce-aicore-addr-transform") + endif() +endif() + +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/runtime/lib64 + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/simulator/${ASCEND_PRODUCT_TYPE}/lib + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/${ASCEND_PRODUCT_TYPE} +) + +# link library +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES stdc++) +if(ASCEND_RUN_MODE STREQUAL "npu") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime) +elseif(ASCEND_RUN_MODE STREQUAL "sim") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES ) + if(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES pem_davinci) + endif() + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime_camodel) +elseif(ASCEND_RUN_MODE STREQUAL "cpu") + message(STATUS "RUN_MODE is cpu") +else() + message(FATAL_ERROR + "ASCEND_RUN_MODE: ${ASCEND_RUN_MODE}\n" + "ASCEND_RUN_MODE must be one of the following: cpu, npu or sim" + ) +endif() +list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES ascendcl) + +set(__IMPLICIT_LINKS) +foreach(dir ${_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES}) + string(APPEND __IMPLICIT_LINKS " -L\"${dir}\"") +endforeach() +foreach(lib ${_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES}) + if(${lib} MATCHES "/") + string(APPEND __IMPLICIT_LINKS " \"${lib}\"") + else() + string(APPEND __IMPLICIT_LINKS " -l${lib}") + endif() +endforeach() + +set(_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/acllib/include + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/include +) +set(__IMPLICIT_INCLUDES) +foreach(inc ${_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES}) + string(APPEND __IMPLICIT_INCLUDES " -I\"${inc}\"") +endforeach() \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake new file mode 100644 index 000000000..f00f227c1 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake @@ -0,0 +1 @@ +set(CMAKE_CCE_COMPILER_WORKS 1 CACHE INTERNAL "") diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt new file mode 100644 index 000000000..661230b17 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt @@ -0,0 +1,37 @@ +# cpu +if (NOT DEFINED ENV{CMAKE_PREFIX_PATH}) + set(CMAKE_PREFIX_PATH ${ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/cmake) +endif() + +find_package(tikicpulib REQUIRED) + +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) + +add_executable(${smoke_testcase}_cpu + ${SRC_FILES} +) + +target_include_directories(${smoke_testcase}_cpu PRIVATE + ${ASCEND_INSTALL_PATH}/acllib/include + ${CMAKE_SOURCE_DIR} +) + +target_link_libraries(${smoke_testcase}_cpu PRIVATE + tikicpulib::${ASCEND_PRODUCT_TYPE} + ascendcl +) + +target_compile_options(${smoke_testcase}_cpu PRIVATE + -g +) + +target_compile_definitions(${smoke_testcase}_cpu PRIVATE + _GLIBCXX_USE_CXX11_ABI=0 +) + +set_target_properties(${smoke_testcase}_cpu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt new file mode 100644 index 000000000..fb1786c6b --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt @@ -0,0 +1,26 @@ +# npu +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) +set_source_files_properties(${SRC_FILES} PROPERTIES LANGUAGE CCE) + +add_executable(${smoke_testcase}_npu + ${SRC_FILES} +) + +target_compile_options(${smoke_testcase}_npu PRIVATE + -O2 + -std=c++17 +) + +target_link_directories(${smoke_testcase}_npu PRIVATE + ${ASCEND_INSTALL_PATH}/lib64 +) + +set_target_properties(${smoke_testcase}_npu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) + +add_custom_target(${smoke_testcase}_sim ALL) +add_dependencies(${smoke_testcase}_sim ${smoke_testcase}_npu) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/data_utils.h b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/data_utils.h new file mode 100644 index 000000000..7cdefd866 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/data_utils.h @@ -0,0 +1,203 @@ +/** + * @file data_utils.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef DATA_UTILS_H +#define DATA_UTILS_H +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +typedef enum { + DT_UNDEFINED = -1, + FLOAT = 0, + HALF = 1, + INT8_T = 2, + INT32_T = 3, + UINT8_T = 4, + INT16_T = 6, + UINT16_T = 7, + UINT32_T = 8, + INT64_T = 9, + UINT64_T = 10, + DOUBLE = 11, + BOOL = 12, + STRING = 13, + COMPLEX64 = 16, + COMPLEX128 = 17, + BF16 = 27 +} printDataType; + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stdout, "[ERROR] " fmt "\n", ##args) +#define CHECK_ACL(x) \ + do { \ + aclError __ret = x; \ + if (__ret != ACL_ERROR_NONE) { \ + std::cerr << __FILE__ << ":" << __LINE__ << " aclError:" << __ret << std::endl; \ + } \ + } while (0); + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t &fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file"); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintHalfData(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(6) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, printDataType dataType, size_t elementsPerRow = 16) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case HALF: + DoPrintHalfData(reinterpret_cast(data), count, elementsPerRow); + break; + case FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } + std::cout << std::endl; +} +#endif // DATA_UTILS_H diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp new file mode 100644 index 000000000..bd760f1c6 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp @@ -0,0 +1,82 @@ +/** + * @file main.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "data_utils.h" +#ifndef ASCENDC_CPU_DEBUG +#include "acl/acl.h" +extern void add_custom_do(uint32_t coreDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z); +#else +#include "tikicpulib.h" +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z); +#endif + +int32_t main(int32_t argc, char *argv[]) +{ + uint32_t blockDim = 1; + size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + +#ifdef ASCENDC_CPU_DEBUG + uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *y = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *z = (uint8_t *)AscendC::GmAlloc(outputByteSize); + + ReadFile("./input/input_x.bin", inputByteSize, x, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, y, inputByteSize); + + AscendC::SetKernelMode(KernelMode::AIV_MODE); + ICPU_RUN_KF(add_custom, blockDim, x, y, z); // use this macro for cpu debug + + WriteFile("./output/output_z.bin", z, outputByteSize); + + AscendC::GmFree((void *)x); + AscendC::GmFree((void *)y); + AscendC::GmFree((void *)z); +#else + CHECK_ACL(aclInit("./scripts/acl.json")); + int32_t deviceId = 0; + CHECK_ACL(aclrtSetDevice(deviceId)); + aclrtStream stream = nullptr; + CHECK_ACL(aclrtCreateStream(&stream)); + + uint8_t *xHost, *yHost, *zHost; + uint8_t *xDevice, *yDevice, *zDevice; + + CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&zHost), outputByteSize)); + CHECK_ACL(aclrtMalloc((void **)&xDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&yDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&zDevice, outputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input/input_x.bin", inputByteSize, xHost, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, yHost, inputByteSize); + + CHECK_ACL(aclrtMemcpy(xDevice, inputByteSize, xHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + CHECK_ACL(aclrtMemcpy(yDevice, inputByteSize, yHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + + add_custom_do(blockDim, nullptr, stream, xDevice, yDevice, zDevice); + CHECK_ACL(aclrtSynchronizeStream(stream)); + + CHECK_ACL(aclrtMemcpy(zHost, outputByteSize, zDevice, outputByteSize, ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output/output_z.bin", zHost, outputByteSize); + + CHECK_ACL(aclrtFree(xDevice)); + CHECK_ACL(aclrtFree(yDevice)); + CHECK_ACL(aclrtFree(zDevice)); + CHECK_ACL(aclrtFreeHost(xHost)); + CHECK_ACL(aclrtFreeHost(yHost)); + CHECK_ACL(aclrtFreeHost(zHost)); + + CHECK_ACL(aclrtDestroyStream(stream)); + CHECK_ACL(aclrtResetDevice(deviceId)); + CHECK_ACL(aclFinalize()); +#endif + return 0; +} diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/run.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/run.sh new file mode 100644 index 000000000..e741e1224 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/run.sh @@ -0,0 +1,130 @@ +#!/bin/bash +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +declare -A VersionMap +VersionMap["Ascend910A"]="Ascend910A" +VersionMap["Ascend910B"]="Ascend910A" +VersionMap["Ascend910ProA"]="Ascend910A" +VersionMap["Ascend910ProB"]="Ascend910A" +VersionMap["Ascend910PremiumA"]="Ascend910A" +VersionMap["Ascend310B1"]="Ascend310B1" +VersionMap["Ascend310B2"]="Ascend310B1" +VersionMap["Ascend310B3"]="Ascend310B1" +VersionMap["Ascend310B4"]="Ascend310B1" +VersionMap["Ascend310P1"]="Ascend310P1" +VersionMap["Ascend310P3"]="Ascend310P1" +VersionMap["Ascend910B1"]="Ascend910B1" +VersionMap["Ascend910B2"]="Ascend910B1" +VersionMap["Ascend910B3"]="Ascend910B1" +VersionMap["Ascend910B4"]="Ascend910B1" +# legacy +VersionMap["ascend910"]="Ascend910A" +VersionMap["ascend310p"]="Ascend310P1" +VersionMap["ascend310B1"]="Ascend310B1" +VersionMap["ascend910B1"]="Ascend910B1" + +FILE_NAME="add" + +SHORT=r:,v:,i:, +LONG=run-mode:,soc-version:,install-path:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" + +while :; do + case "$1" in + -r | --run-mode) + RUN_MODE="$2" + shift 2 + ;; + -v | --soc-version) + SOC_VERSION="$2" + shift 2 + ;; + -i | --install-path) + ASCEND_INSTALL_PATH="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +# in case of running op in simulator, use stub so instead +if [ "${RUN_MODE}" = "sim" ]; then + export LD_LIBRARY_PATH=$_ASCEND_INSTALL_PATH/runtime/lib64/stub:$LD_LIBRARY_PATH + if [ ! $CAMODEL_LOG_PATH ]; then + export CAMODEL_LOG_PATH=./sim_log + fi + rm -rf $CAMODEL_LOG_PATH + mkdir -p $CAMODEL_LOG_PATH +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash + +if [[ " ${!VersionMap[*]} " != *" $SOC_VERSION "* ]]; then + echo "ERROR: SOC_VERSION should be in [${!VersionMap[*]}]" + exit -1 +fi +_SOC_VERSION=${VersionMap[$SOC_VERSION]} + +if [ $_SOC_VERSION"x" = "Ascend910Ax" ] || [ $_SOC_VERSION"x" = "Ascend310P1x" ] || [ $_SOC_VERSION"x" = "Ascend310B1x" ]; then + CORE_TYPE="AiCore" +elif [ $_SOC_VERSION"x" = "Ascend910B1x" ]; then + CORE_TYPE="VectorCore" +fi + +RUN_MODE_LIST="cpu sim npu" +if [[ " $RUN_MODE_LIST " != *" $RUN_MODE "* ]]; then + echo "ERROR: RUN_MODE error, This sample only support specify cpu, sim or npu!" + exit -1 +fi + +set -e +rm -rf build *_cpu *_sim *_npu cceprint npuchk *log *.vcd + +rm -rf build +mkdir -p build +cmake -B build \ + -Dsmoke_testcase=${FILE_NAME} \ + -DASCEND_PRODUCT_TYPE=${_SOC_VERSION} \ + -DASCEND_CORE_TYPE=${CORE_TYPE} \ + -DASCEND_RUN_MODE=${RUN_MODE} \ + -DASCEND_INSTALL_PATH=${_ASCEND_INSTALL_PATH} +cmake --build build --target ${FILE_NAME}_${RUN_MODE} +if [ $? -ne 0 ]; then + echo "ERROR: compile op on failed!" + exit -1 +fi +echo "INFO: compile op on ${RUN_MODE} succeed!" + +rm -rf input/*.bin output/*.bin +python3 scripts/gen_data.py +(export LD_LIBRARY_PATH=${_ASCEND_INSTALL_PATH}/tools/simulator/${_SOC_VERSION}/lib:$LD_LIBRARY_PATH && ./${FILE_NAME}_${RUN_MODE}) +if [ $? -ne 0 ]; then + echo "ERROR: execute op on ${RUN_MODE} failed!" + exit -1 +fi +echo "INFO: execute op on ${RUN_MODE} succeed!" +python3 scripts/verify_result.py output/output_z.bin output/golden.bin + +rm -rf *log *.vcd diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/acl.json b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/gen_data.py new file mode 100644 index 000000000..a85a84a15 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/gen_data.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/verify_result.py new file mode 100644 index 000000000..ddb6e4a31 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 0: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/README.md b/operator_contrib/CommonMistake/AclTensorFreeFail/README.md new file mode 100644 index 000000000..2bf047445 --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/README.md @@ -0,0 +1,45 @@ +## 样例说明 +该样例用于复现aclnn调用算子时，输出释放失败错误 + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` +INFO: execute op! +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Copy input[1] success +[INFO] Create stream success +[INFO] Execute GetWorkspaceSize success, workspace size 0 +[INFO] Execute Operator success +[INFO] Synchronize stream success +[INFO] Copy output[0] success +[INFO] Write output success +[INFO] Run op success +[ERROR] Free devOutputs[0]error code is 507899 + + +``` + +使用grep在plog日志查找ERROR关键字有类似信息 + +:[ERROR] ASCENDCL(3188519,execute_op):2024-08-14-18:40:38.729.998 [memory.cpp:229]3188519 aclrtFree: free device memory failed, runtime result = 507899 + + +## 解决思路 +1、阅读代码，根据屏幕打印的输出下标查看对应内存使用是否正确，例如Free devOutputs[0]error就需要看第1个输出内存使用是否正常。 +2、通过添加printf打印，直接在输出各个API的地址偏移（需要产品支持printf调试API） +3、构造相同输入，通过孪生调试功能的CPU侧调试进行定位，该方式可以直接显示错误原因，然后根据错误原因做具体分析，本例会输出类似： + +[ERROR][CORE_0][pid 3576780] error happened! ========= +**SIGSEGV Signal (Invalid memory reference) catched,** backtrace info: +[#0] 0x000000000041c678: handler(int) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/lib/include/kern_fwk.h:98 +[#1] 0x0000000000416cc8: **void AscendC::DataCopyUB2GMImpl**(float16::Fp16T*, float16::Fp16T*, AscendC::DataCopyParams const&) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/dav_c220/kernel_operator_data_copy_impl.h:125 +[#2] 0x0000000000414044: void AscendC::DataCopy(AscendC::GlobalTensor const&, AscendC::LocalTensor const&, AscendC::DataCopyParams const&) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/inner_interface/inner_kernel_operator_data_copy_intf.cppm:155 +[#3] 0x0000000000410e1c: void AscendC::DataCopy(AscendC::GlobalTensor const&, AscendC::LocalTensor const&, unsigned int) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/inner_interface/inner_kernel_operator_data_copy_intf.cppm:460 + +根据加粗内容可知，代码中使用了无效内存，结合下面的信息可得该无效内存在调用DataCopy，把LocalTensor搬运到GlobalTensor时使用。根据报错信息可以重点检查对应搬运通路的DataCopy的GM使用是否正常。 + diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh new file mode 100755 index 000000000..dde6963ce --- /dev/null +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +LOG_PATH=/var/log/npu/slog/debug +if [ -d "$LOG_PATH" ]; then + echo "清理日志，路径：$LOG_PATH" +else + LOG_PATH=~/ascend/log + echo "清理日志，路径：$LOG_PATH" +fi +rm -rf $LOG_PATH* +cd AddCustom;./build.sh;./build_out/*.run +cd ../AclNNInvocation;bash run.sh +cd $LOG_PATH; grep -rn ERROR diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/README.md b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/README.md new file mode 100644 index 000000000..75fa533f7 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/README.md @@ -0,0 +1,54 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ └── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ```cpp + // 获取算子使用的workspace空间大小 + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + // 执行算子 + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ```bash + bash run.sh + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2024/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..11bb4aeca --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** + * @file common.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..66b6ea2c3 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,180 @@ +/** + * @file op_runner.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "acl/acl.h" +#include "aclnn/acl_meta.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..6d8ee0905 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** + * @file operator_desc.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/run.sh new file mode 100644 index 000000000..25433eb68 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while :; do + case "$1" in + # float16, float, int32 + -v | --dtype) + DTYPE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash +export DDK_PATH=$_ASCEND_INSTALL_PATH +export NPU_HOST_LIB=$_ASCEND_INSTALL_PATH/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR + rm -rf build + mkdir -p build + cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_add_op + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output_z.bin output/golden.bin + if [ $? -ne 0 ]; then + echo "ERROR: verify result failed!" + return 1 + fi +} + +main diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..92b5a5bcf --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..ddb6e4a31 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 0: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..992759c95 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/common.cpp @@ -0,0 +1,80 @@ +/** + * @file common.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "common.h" + +#include +#include +#include + +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + size_t writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..481a31df8 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/main.cpp @@ -0,0 +1,162 @@ +/** + * @file main.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include +#include +#include + +#include +#include + +#include "acl/acl.h" +#include "common.h" +#include "op_runner.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..7e1623aed --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,453 @@ +/** + * @file op_runner.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "op_runner.h" + +#include +#include + +#include "acl/acl_op_compiler.h" +#include "aclnn_add_custom.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = + aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), nullptr, 0, + GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = + aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), nullptr, 0, + GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = + aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..da04cf6c9 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,51 @@ +/** + * @file operator_desc.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "operator_desc.h" + +#include "common.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/build.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..6fc77c881 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,73 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 1; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..323f3076f --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,22 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..6d2c1346c --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,96 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * this->tileLength], zLocal[1], this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, + uint8_t *workspace, uint8_t *tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/CMakeLists.txt new file mode 100644 index 000000000..fd87c7620 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.16) +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_CXX_STANDARD 17) + +set(CCE_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules) +list(APPEND CMAKE_MODULE_PATH ${CCE_CMAKE_PATH}) +project(kernel_samples LANGUAGES CCE CXX) + +add_subdirectory(cmake/cpu) +add_subdirectory(cmake/npu) \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/README.md b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/README.md new file mode 100644 index 000000000..49bee5787 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/README.md @@ -0,0 +1,81 @@ +## 目录结构介绍 +``` +├── AddKernelInvocation +│ ├── cmake // 编译工程文件 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 验证输出数据和真值数据是否一致的验证脚本 +│ │ └── verify_result.py // 真值对比文件 +│ │── add_custom.cpp // 算子kernel实现 +│ │── CMakeLists.txt // 编译工程文件 +│ │── data_utils.h // 数据读入写出函数 +│ │── main.cpp // 主函数，调用算子的应用程序，含CPU域及NPU域调用 +│ └── run.sh // 编译运行算子的脚本 +``` +## 代码实现介绍 +本调用样例中实现的是固定shape为8*2048的Add算子。 +- kernel实现 + Add算子的数学表达式为： + ``` + z = x + y + ``` + 计算逻辑是：Ascend C提供的矢量计算接口的操作元素都为LocalTensor，输入数据需要先搬运进片上存储，然后使用计算接口完成两个输入参数相加，得到最终结果，再搬出到外部存储上。 + + Add算子的实现流程分为3个基本任务：CopyIn，Compute，CopyOut。CopyIn任务负责将Global Memory上的输入Tensor xGm和yGm搬运到Local Memory，分别存储在xLocal、yLocal，Compute任务负责对xLocal、yLocal执行加法操作，计算结果存储在zLocal中，CopyOut任务负责将输出数据从zLocal搬运至Global Memory上的输出Tensor zGm中。具体请参考[add_custom.cpp](./add_custom.cpp)。 + +- 调用实现 + 1. CPU侧运行验证主要通过ICPU_RUN_KF CPU调测宏等CPU调测库提供的接口来完成； + 2. NPU侧运行验证主要通过使用<<<>>>内核调用符来完成。 + + 应用程序通过ASCENDC_CPU_DEBUG 宏区分代码逻辑运行于CPU侧还是NPU侧。 + +## 运行样例算子 + - 打开样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/KernelLaunch/AddKernelInvocation + ``` + - 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ```` + + 配置仿真模式日志文件目录，默认为sim_log。 + ```bash + export CAMODEL_LOG_PATH=./sim_log + ``` + + - 样例执行 + + ```bash + bash run.sh -r [RUN_MODE] -v [SOC_VERSION] + ``` + - RUN_MODE：编译方式，可选择CPU调试，NPU仿真，NPU上板。支持参数为[cpu / sim / npu]，默认值为cpu。 + - SOC_VERSION：昇腾AI处理器型号，如果无法确定具体的[SOC_VERSION]，则在安装昇腾AI处理器的服务器执行npu-smi info命令进行查询，在查询到的“Name”前增加Ascend信息，例如“Name”对应取值为xxxyy，实际配置的[SOC_VERSION]值为Ascendxxxyy。支持以下参数取值（xxx请替换为具体取值）： + - Atlas 推理系列产品（Ascend 310P处理器）参数值：Ascend310P1、Ascend310P3 + - Atlas 训练系列产品参数值：AscendxxxA、AscendxxxB + - Atlas A2训练系列产品参数值：AscendxxxB1、AscendxxxB2、AscendxxxB3、AscendxxxB4 + + 注：针对Atlas 训练系列产品使用NPU仿真调试，会存在精度问题，可选择其他芯片进行NPU仿真调试。 + + 示例如下。 + ```bash + bash run.sh -r cpu -v Ascend310P1 + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2023/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp new file mode 100644 index 000000000..d77c14b83 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp @@ -0,0 +1,91 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; + +constexpr int32_t TOTAL_LENGTH = 8 * 2048; // total length of data +constexpr int32_t USE_CORE_NUM = 1; // num of core used +constexpr int32_t BLOCK_LENGTH = TOTAL_LENGTH / USE_CORE_NUM; // length computed of each core +constexpr int32_t TILE_NUM = 8; // split data into 8 tiles for each core +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue +constexpr int32_t TILE_LENGTH = BLOCK_LENGTH / TILE_NUM / BUFFER_NUM; // seperate to 2 parts, due to double buffer + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z) + { + xGm.SetGlobalBuffer((__gm__ half *)x + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + yGm.SetGlobalBuffer((__gm__ half *)y + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + zGm.SetGlobalBuffer((__gm__ half *)z + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + pipe.InitBuffer(inQueueX, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + } + __aicore__ inline void Process() + { + int32_t loopCount = TILE_NUM * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * TILE_LENGTH], TILE_LENGTH); + DataCopy(yLocal, yGm[progress * TILE_LENGTH], TILE_LENGTH); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, TILE_LENGTH); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * TILE_LENGTH], zLocal[1], TILE_LENGTH); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +{ + KernelAdd op; + op.Init(x, y, z); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +{ + add_custom<<>>(x, y, z); +} +#endif diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in new file mode 100644 index 000000000..a9b5688ff --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in @@ -0,0 +1,5 @@ +set(CMAKE_CCE_COMPILER "@CMAKE_CCE_COMPILER@") +set(CMAKE_CCE_COMPILER_LOADED 1) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS @CMAKE_CCE_SOURCE_FILE_EXTENSIONS@) +set(CMAKE_CCE_OUTPUT_EXTENSION @CMAKE_CCE_OUTPUT_EXTENSION@) +set(CMAKE_CCE_COMPILER_ENV_VAR "@CMAKE_CCE_COMPILER_ENV_VAR@") diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake new file mode 100644 index 000000000..7d2fc2ddd --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake @@ -0,0 +1,41 @@ +include(CMakeCommonLanguageInclude) + +set(CMAKE_INCLUDE_FLAG_CCE "-I") + +if(UNIX) + set(CMAKE_CCE_OUTPUT_EXTENSION .o) +else() + set(CMAKE_CCE_OUTPUT_EXTENSION .obj) +endif() + +set(_INCLUDED_FILE 0) +set(CMAKE_SHARED_LIBRARY_CCE_FLAGS -fPIC) +set(CMAKE_SHARED_LIBRARY_CREATE_CCE_FLAGS -shared) +set(CMAKE_STATIC_LIBRARY_CREATE_CCE_FLAGS "--cce-build-static-lib") +set(CMAKE_LIBRARY_CREATE_CCE_FLAGS "--cce-fatobj-link") + +if(NOT CMAKE_CCE_COMPILE_OBJECT) + set(CMAKE_CCE_COMPILE_OBJECT + " -xcce ${__IMPLICIT_INCLUDES} ${_CMAKE_CCE_BUILTIN_INCLUDE_PATH} ${_CMAKE_COMPILE_AS_CCE_FLAG} ${_CMAKE_CCE_COMPILE_OPTIONS} ${_CMAKE_CCE_COMMON_COMPILE_OPTIONS} -pthread -o -c ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_LIBRARY) + set(CMAKE_CCE_CREATE_SHARED_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_STATIC_LIBRARY) + set(CMAKE_CCE_CREATE_STATIC_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_MODULE) + set(CMAKE_CCE_CREATE_SHARED_MODULE ${CMAKE_CCE_CREATE_SHARED_LIBRARY}) +endif() + +if(NOT CMAKE_CCE_LINK_EXECUTABLE) + set(CMAKE_CCE_LINK_EXECUTABLE + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ${__IMPLICIT_LINKS}") +endif() + +set(CMAKE_CCE_INFORMATION_LOADED 1) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake new file mode 100755 index 000000000..f3c0a4732 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake @@ -0,0 +1,124 @@ +find_program(CMAKE_CCE_COMPILER NAMES "ccec" PATHS "$ENV{PATH}" DOC "CCE Compiler") + +mark_as_advanced(CMAKE_CCE_COMPILER) + +message(STATUS "CMAKE_CCE_COMPILER: " ${CMAKE_CCE_COMPILER}) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS cce;cpp) +set(CMAKE_CCE_COMPILER_ENV_VAR "CCE") +message(STATUS "CMAKE_CURRENT_LIST_DIR: " ${CMAKE_CURRENT_LIST_DIR}) + +# configure all variables set in this file +configure_file(${CMAKE_CURRENT_LIST_DIR}/CMakeCCECompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeCCECompiler.cmake + @ONLY +) + +message(STATUS "ASCEND_PRODUCT_TYPE:\n" " ${ASCEND_PRODUCT_TYPE}") +message(STATUS "ASCEND_CORE_TYPE:\n" " ${ASCEND_CORE_TYPE}") +message(STATUS "ASCEND_INSTALL_PATH:\n" " ${ASCEND_INSTALL_PATH}") + +if(DEFINED ASCEND_INSTALL_PATH) + set(_CMAKE_ASCEND_INSTALL_PATH ${ASCEND_INSTALL_PATH}) +else() + message(FATAL_ERROR + "no, installation path found, should passing -DASCEND_INSTALL_PATH= in cmake" + ) + set(_CMAKE_ASCEND_INSTALL_PATH) +endif() + + +if(DEFINED ASCEND_PRODUCT_TYPE) + set(_CMAKE_CCE_COMMON_COMPILE_OPTIONS "--cce-auto-sync -mllvm -api-deps-filter") + if(ASCEND_PRODUCT_TYPE STREQUAL "") + message(FATAL_ERROR "ASCEND_PRODUCT_TYPE must be non-empty if set.") + elseif(ASCEND_PRODUCT_TYPE AND NOT ASCEND_PRODUCT_TYPE MATCHES "^Ascend[0-9][0-9][0-9][a-zA-Z]?[1-9]?$") + message(FATAL_ERROR + "ASCEND_PRODUCT_TYPE: ${ASCEND_PRODUCT_TYPE}\n" + "is not one of the following: Ascend910A, Ascend310P1, Ascend910B1, Ascend310B1" + ) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c100") + else() + message(FATAL_ERROR, "only AiCore inside") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310P1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-cube") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-record-overflow=false -mllvm -cce-aicore-addr-transform") + endif() +endif() + +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/runtime/lib64 + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/simulator/${ASCEND_PRODUCT_TYPE}/lib + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/${ASCEND_PRODUCT_TYPE} +) + +# link library +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES stdc++) +if(ASCEND_RUN_MODE STREQUAL "npu") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime) +elseif(ASCEND_RUN_MODE STREQUAL "sim") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES ) + if(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES pem_davinci) + endif() + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime_camodel) +elseif(ASCEND_RUN_MODE STREQUAL "cpu") + message(STATUS "RUN_MODE is cpu") +else() + message(FATAL_ERROR + "ASCEND_RUN_MODE: ${ASCEND_RUN_MODE}\n" + "ASCEND_RUN_MODE must be one of the following: cpu, npu or sim" + ) +endif() +list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES ascendcl) + +set(__IMPLICIT_LINKS) +foreach(dir ${_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES}) + string(APPEND __IMPLICIT_LINKS " -L\"${dir}\"") +endforeach() +foreach(lib ${_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES}) + if(${lib} MATCHES "/") + string(APPEND __IMPLICIT_LINKS " \"${lib}\"") + else() + string(APPEND __IMPLICIT_LINKS " -l${lib}") + endif() +endforeach() + +set(_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/acllib/include + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/include +) +set(__IMPLICIT_INCLUDES) +foreach(inc ${_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES}) + string(APPEND __IMPLICIT_INCLUDES " -I\"${inc}\"") +endforeach() \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake new file mode 100644 index 000000000..f00f227c1 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake @@ -0,0 +1 @@ +set(CMAKE_CCE_COMPILER_WORKS 1 CACHE INTERNAL "") diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/cpu/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/cpu/CMakeLists.txt new file mode 100644 index 000000000..661230b17 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/cpu/CMakeLists.txt @@ -0,0 +1,37 @@ +# cpu +if (NOT DEFINED ENV{CMAKE_PREFIX_PATH}) + set(CMAKE_PREFIX_PATH ${ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/cmake) +endif() + +find_package(tikicpulib REQUIRED) + +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) + +add_executable(${smoke_testcase}_cpu + ${SRC_FILES} +) + +target_include_directories(${smoke_testcase}_cpu PRIVATE + ${ASCEND_INSTALL_PATH}/acllib/include + ${CMAKE_SOURCE_DIR} +) + +target_link_libraries(${smoke_testcase}_cpu PRIVATE + tikicpulib::${ASCEND_PRODUCT_TYPE} + ascendcl +) + +target_compile_options(${smoke_testcase}_cpu PRIVATE + -g +) + +target_compile_definitions(${smoke_testcase}_cpu PRIVATE + _GLIBCXX_USE_CXX11_ABI=0 +) + +set_target_properties(${smoke_testcase}_cpu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/npu/CMakeLists.txt b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/npu/CMakeLists.txt new file mode 100644 index 000000000..fb1786c6b --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/cmake/npu/CMakeLists.txt @@ -0,0 +1,26 @@ +# npu +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) +set_source_files_properties(${SRC_FILES} PROPERTIES LANGUAGE CCE) + +add_executable(${smoke_testcase}_npu + ${SRC_FILES} +) + +target_compile_options(${smoke_testcase}_npu PRIVATE + -O2 + -std=c++17 +) + +target_link_directories(${smoke_testcase}_npu PRIVATE + ${ASCEND_INSTALL_PATH}/lib64 +) + +set_target_properties(${smoke_testcase}_npu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) + +add_custom_target(${smoke_testcase}_sim ALL) +add_dependencies(${smoke_testcase}_sim ${smoke_testcase}_npu) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/data_utils.h b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/data_utils.h new file mode 100644 index 000000000..7cdefd866 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/data_utils.h @@ -0,0 +1,203 @@ +/** + * @file data_utils.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef DATA_UTILS_H +#define DATA_UTILS_H +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +typedef enum { + DT_UNDEFINED = -1, + FLOAT = 0, + HALF = 1, + INT8_T = 2, + INT32_T = 3, + UINT8_T = 4, + INT16_T = 6, + UINT16_T = 7, + UINT32_T = 8, + INT64_T = 9, + UINT64_T = 10, + DOUBLE = 11, + BOOL = 12, + STRING = 13, + COMPLEX64 = 16, + COMPLEX128 = 17, + BF16 = 27 +} printDataType; + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stdout, "[ERROR] " fmt "\n", ##args) +#define CHECK_ACL(x) \ + do { \ + aclError __ret = x; \ + if (__ret != ACL_ERROR_NONE) { \ + std::cerr << __FILE__ << ":" << __LINE__ << " aclError:" << __ret << std::endl; \ + } \ + } while (0); + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t &fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file"); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintHalfData(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(6) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, printDataType dataType, size_t elementsPerRow = 16) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case HALF: + DoPrintHalfData(reinterpret_cast(data), count, elementsPerRow); + break; + case FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } + std::cout << std::endl; +} +#endif // DATA_UTILS_H diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp new file mode 100644 index 000000000..bd760f1c6 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp @@ -0,0 +1,82 @@ +/** + * @file main.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "data_utils.h" +#ifndef ASCENDC_CPU_DEBUG +#include "acl/acl.h" +extern void add_custom_do(uint32_t coreDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z); +#else +#include "tikicpulib.h" +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z); +#endif + +int32_t main(int32_t argc, char *argv[]) +{ + uint32_t blockDim = 1; + size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + +#ifdef ASCENDC_CPU_DEBUG + uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *y = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *z = (uint8_t *)AscendC::GmAlloc(outputByteSize); + + ReadFile("./input/input_x.bin", inputByteSize, x, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, y, inputByteSize); + + AscendC::SetKernelMode(KernelMode::AIV_MODE); + ICPU_RUN_KF(add_custom, blockDim, x, y, z); // use this macro for cpu debug + + WriteFile("./output/output_z.bin", z, outputByteSize); + + AscendC::GmFree((void *)x); + AscendC::GmFree((void *)y); + AscendC::GmFree((void *)z); +#else + CHECK_ACL(aclInit("./scripts/acl.json")); + int32_t deviceId = 0; + CHECK_ACL(aclrtSetDevice(deviceId)); + aclrtStream stream = nullptr; + CHECK_ACL(aclrtCreateStream(&stream)); + + uint8_t *xHost, *yHost, *zHost; + uint8_t *xDevice, *yDevice, *zDevice; + + CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&zHost), outputByteSize)); + CHECK_ACL(aclrtMalloc((void **)&xDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&yDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&zDevice, outputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input/input_x.bin", inputByteSize, xHost, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, yHost, inputByteSize); + + CHECK_ACL(aclrtMemcpy(xDevice, inputByteSize, xHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + CHECK_ACL(aclrtMemcpy(yDevice, inputByteSize, yHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + + add_custom_do(blockDim, nullptr, stream, xDevice, yDevice, zDevice); + CHECK_ACL(aclrtSynchronizeStream(stream)); + + CHECK_ACL(aclrtMemcpy(zHost, outputByteSize, zDevice, outputByteSize, ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output/output_z.bin", zHost, outputByteSize); + + CHECK_ACL(aclrtFree(xDevice)); + CHECK_ACL(aclrtFree(yDevice)); + CHECK_ACL(aclrtFree(zDevice)); + CHECK_ACL(aclrtFreeHost(xHost)); + CHECK_ACL(aclrtFreeHost(yHost)); + CHECK_ACL(aclrtFreeHost(zHost)); + + CHECK_ACL(aclrtDestroyStream(stream)); + CHECK_ACL(aclrtResetDevice(deviceId)); + CHECK_ACL(aclFinalize()); +#endif + return 0; +} diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/run.sh b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/run.sh new file mode 100644 index 000000000..e741e1224 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/run.sh @@ -0,0 +1,130 @@ +#!/bin/bash +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +declare -A VersionMap +VersionMap["Ascend910A"]="Ascend910A" +VersionMap["Ascend910B"]="Ascend910A" +VersionMap["Ascend910ProA"]="Ascend910A" +VersionMap["Ascend910ProB"]="Ascend910A" +VersionMap["Ascend910PremiumA"]="Ascend910A" +VersionMap["Ascend310B1"]="Ascend310B1" +VersionMap["Ascend310B2"]="Ascend310B1" +VersionMap["Ascend310B3"]="Ascend310B1" +VersionMap["Ascend310B4"]="Ascend310B1" +VersionMap["Ascend310P1"]="Ascend310P1" +VersionMap["Ascend310P3"]="Ascend310P1" +VersionMap["Ascend910B1"]="Ascend910B1" +VersionMap["Ascend910B2"]="Ascend910B1" +VersionMap["Ascend910B3"]="Ascend910B1" +VersionMap["Ascend910B4"]="Ascend910B1" +# legacy +VersionMap["ascend910"]="Ascend910A" +VersionMap["ascend310p"]="Ascend310P1" +VersionMap["ascend310B1"]="Ascend310B1" +VersionMap["ascend910B1"]="Ascend910B1" + +FILE_NAME="add" + +SHORT=r:,v:,i:, +LONG=run-mode:,soc-version:,install-path:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" + +while :; do + case "$1" in + -r | --run-mode) + RUN_MODE="$2" + shift 2 + ;; + -v | --soc-version) + SOC_VERSION="$2" + shift 2 + ;; + -i | --install-path) + ASCEND_INSTALL_PATH="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +# in case of running op in simulator, use stub so instead +if [ "${RUN_MODE}" = "sim" ]; then + export LD_LIBRARY_PATH=$_ASCEND_INSTALL_PATH/runtime/lib64/stub:$LD_LIBRARY_PATH + if [ ! $CAMODEL_LOG_PATH ]; then + export CAMODEL_LOG_PATH=./sim_log + fi + rm -rf $CAMODEL_LOG_PATH + mkdir -p $CAMODEL_LOG_PATH +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash + +if [[ " ${!VersionMap[*]} " != *" $SOC_VERSION "* ]]; then + echo "ERROR: SOC_VERSION should be in [${!VersionMap[*]}]" + exit -1 +fi +_SOC_VERSION=${VersionMap[$SOC_VERSION]} + +if [ $_SOC_VERSION"x" = "Ascend910Ax" ] || [ $_SOC_VERSION"x" = "Ascend310P1x" ] || [ $_SOC_VERSION"x" = "Ascend310B1x" ]; then + CORE_TYPE="AiCore" +elif [ $_SOC_VERSION"x" = "Ascend910B1x" ]; then + CORE_TYPE="VectorCore" +fi + +RUN_MODE_LIST="cpu sim npu" +if [[ " $RUN_MODE_LIST " != *" $RUN_MODE "* ]]; then + echo "ERROR: RUN_MODE error, This sample only support specify cpu, sim or npu!" + exit -1 +fi + +set -e +rm -rf build *_cpu *_sim *_npu cceprint npuchk *log *.vcd + +rm -rf build +mkdir -p build +cmake -B build \ + -Dsmoke_testcase=${FILE_NAME} \ + -DASCEND_PRODUCT_TYPE=${_SOC_VERSION} \ + -DASCEND_CORE_TYPE=${CORE_TYPE} \ + -DASCEND_RUN_MODE=${RUN_MODE} \ + -DASCEND_INSTALL_PATH=${_ASCEND_INSTALL_PATH} +cmake --build build --target ${FILE_NAME}_${RUN_MODE} +if [ $? -ne 0 ]; then + echo "ERROR: compile op on failed!" + exit -1 +fi +echo "INFO: compile op on ${RUN_MODE} succeed!" + +rm -rf input/*.bin output/*.bin +python3 scripts/gen_data.py +(export LD_LIBRARY_PATH=${_ASCEND_INSTALL_PATH}/tools/simulator/${_SOC_VERSION}/lib:$LD_LIBRARY_PATH && ./${FILE_NAME}_${RUN_MODE}) +if [ $? -ne 0 ]; then + echo "ERROR: execute op on ${RUN_MODE} failed!" + exit -1 +fi +echo "INFO: execute op on ${RUN_MODE} succeed!" +python3 scripts/verify_result.py output/output_z.bin output/golden.bin + +rm -rf *log *.vcd diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/acl.json b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/gen_data.py new file mode 100644 index 000000000..a85a84a15 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/gen_data.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/verify_result.py new file mode 100644 index 000000000..1a21d809a --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 100: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/AddressMisalign/README.md b/operator_contrib/CommonMistake/AddressMisalign/README.md new file mode 100644 index 000000000..ee473bd25 --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/README.md @@ -0,0 +1,52 @@ +## 样例说明 +该样例用于复现算子UB非对齐错误 + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` + + [INFO] Set device[0] success + [INFO] Get RunMode[0] success + [INFO] Init resource success + [INFO] Set input success + [INFO] Copy input[0] success + [INFO] Copy input[1] success + [INFO] Copy input[2] success + [INFO] Create stream success + [INFO] Execute GetWorkspaceSize success, workspace size 0 + [INFO] Execute Operator success + [ERROR] Synchronize stream failed. error code is 507015 (不同产品会不同，可能为507035) + [ERROR] Run op failed + [INFO] Reset Device success + [INFO] Destory resource success + +``` + +使用grep在plog日志查找errorStr关键字有类似信息 + +device-app-3157491/device-app-3157491_20240814113949991.log:534:[ERROR] RUNTIME(3157491,execute_op):2024-08-14-11:39:50.019.104 [device_error_proc.cc:1219]3157491 ProcessStarsCoreErrorInfo:The extend info: errcode:(0, 0x10000, 0) ** errorStr: instruction address misalign(ADDR_MISALIGN). ** fixp_error0 info: 0x7a, fixp_error1 info: 0xe6 fsmId:0, tslot:0, thread:0, ctxid:0, blk:0, sublk:0, subErrType:4. + +或 + +plog/plog-3258665_20240814072045415.log:22411:[ERROR] RUNTIME(3258665,execute_add_op):2024-08-14-07:20:45.937.241 [device_error_proc.cc:1230]3258665 ProcessStarsCoreErrorInfo:The extend info: errcode:(0, 0x800, 0)** errorStr: The UB address accessed by the VEC instruction is not aligned. **fixp_error0 info: 0x300000e, fixp_error1 info: 0xb9 fsmId:0, tslot:4, thread:0, ctxid:0, blk:7, sublk:0, subErrType:4. + + +## 解决思路 +1、阅读代码，检查API使用是否满足通用对齐约束 +2、通过添加printf打印，直接在输出各个API的地址偏移（需要产品支持printf调试API） +3、构造相同输入，通过孪生调试功能的CPU侧调试进行定位，该方式可以直接显示错误行数，本例会输出类似： + +[TmSim]: Run in serial mode. +[ERROR]instr DataCopy(LocalTensor To GlobalTensor) **src addr is 8194, which should be 32 B Aligned** +[ASSERT] /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/inner_interface/inner_kernel_operator_data_copy_intf.cppm:143: Assertion `false && "Check DataCopy(LocalTensor To GlobalTensor) instr failed"' +[ERROR][CORE_0][pid 3283619] error happened! ========= +SIGABRT Signal (Abort Signal from abort) catched, backtrace info: +[#0] 0x000000000041c9c0: handler(int) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/lib/include/kern_fwk.h:98 +[#1] 0x0000000000414334: void AscendC::DataCopy(AscendC::GlobalTensor const&, AscendC::LocalTensor const&, AscendC::DataCopyParams const&) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/inner_interface/inner_kernel_operator_data_copy_intf.cppm:146 +[#2] 0x000000000041108c: void AscendC::DataCopy(AscendC::GlobalTensor const&, AscendC::LocalTensor const&, unsigned int) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/inner_interface/inner_kernel_operator_data_copy_intf.cppm:460 +[#3] 0x000000000040e3f8: KernelAdd::CopyOut(int) **at /root/libotao/ascend-error-example/AddressMisalign/AddKernelInvocation/add_custom.cpp:65 (discriminator 4) ** +[#4] 0x000000000040e14c: KernelAdd::Process() at /root/libotao/ascend-error-example/AddressMisalign/AddKernelInvocation/add_custom.cpp:35 (discriminator 2) +[#5] 0x000000000040ad04: add_custom at /root/libotao/ascend-error-example/AddressMisalign/AddKernelInvocation/add_custom.cpp:80 +[#6] 0x000000000041c568: main at /root/libotao/ascend-error-example/AddressMisalign/AddKernelInvocation/main.cpp:34 (discriminator 127) +[#7] 0x000000000040abf0: _start at ??:? diff --git a/operator_contrib/CommonMistake/AddressMisalign/run_test.sh b/operator_contrib/CommonMistake/AddressMisalign/run_test.sh new file mode 100755 index 000000000..e40ecfb5a --- /dev/null +++ b/operator_contrib/CommonMistake/AddressMisalign/run_test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +LOG_PATH=/var/log/npu/slog/debug +if [ -d "$LOG_PATH" ]; then + echo "清理日志，路径：$LOG_PATH" +else + LOG_PATH=~/ascend/log + echo "清理日志，路径：$LOG_PATH" +fi +rm -rf $LOG_PATH* +cd AddCustom;./build.sh;./build_out/*.run +cd ../AclNNInvocation;bash run.sh +cd $LOG_PATH; grep -rn errorStr \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/README.md b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/README.md new file mode 100755 index 000000000..69d8c877a --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/README.md @@ -0,0 +1,52 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ ├── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ ├── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ ├── operator_desc.cpp // 构造算子的输入与输出描述 +│ ├── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ``` + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ``` + cd $HOME/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ``` + bash run.sh + ``` +## 更新说明 + | 时间 | 更新事项 | +|----|------| +| 2024/5/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/common.h new file mode 100755 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/op_runner.h new file mode 100755 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/operator_desc.h new file mode 100755 index 000000000..4cbdf07e0 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/run.sh new file mode 100755 index 000000000..d7fd76334 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/run.sh @@ -0,0 +1,99 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + ./execute_add_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret=`python3 scripts/verify_result.py output/output_z.bin output/golden.bin` + echo $ret + if [ "x$ret" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/acl.json new file mode 100755 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/gen_data.py new file mode 100755 index 000000000..7babb0e76 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd +import numpy as np +import os +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/verify_result.py new file mode 100755 index 000000000..2528c9f07 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,23 @@ +import os +import sys +import numpy as np + +loss = 1e-3 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.float16) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.float16) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/CMakeLists.txt new file mode 100755 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp new file mode 100755 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/main.cpp new file mode 100755 index 000000000..a72e89598 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/main.cpp @@ -0,0 +1,163 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape { 8, 2048 }; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/op_runner.cpp new file mode 100755 index 000000000..cfad545ff --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,452 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_add_custom.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], + &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/operator_desc.cpp new file mode 100755 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakePresets.json new file mode 100755 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/build.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/config.cmake new file mode 100755 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..a14913c65 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b", "ascend910c"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c8b319944 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910c1": "ascend910c", + "ascend910c2": "ascend910c", + "ascend910c3": "ascend910c", + "ascend910c4": "ascend910c", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100755 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100755 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp new file mode 100755 index 000000000..3262f2c31 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,72 @@ + +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" +#include "graph/utils/type_utils.h" +#include "tiling/platform/platform_ascendc.h" +namespace optiling { +const uint32_t BLOCK_DIM = 1; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + TilingData tiling; + uint64_t ubSize; + auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); + ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); + std::cout<GetInputTensor(0)->GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +static graphStatus InferDataType(gert::InferDataTypeContext* context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType);; + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h new file mode 100755 index 000000000..4bb6d940f --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) + TILING_DATA_FIELD_DEF(uint32_t, totalLength); + TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp new file mode 100755 index 000000000..62b57bd32 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * + * Function : z = x + y + * This sample is a very basic sample that implements vector add on Ascend plaform. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + xGm.SetGlobalBuffer((__gm__ DTYPE_X*)x , 2048); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y*)y , 2048); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z*)z ,2048); + pipe.InitBuffer(inQueueX, BUFFER_NUM, 4000 * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, 4000 * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, 4000 * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < 1; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * 2048], 2048); + DataCopy(yLocal, yGm[progress * 2048], 2048); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, 2048); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm, zLocal, 2048); + zGm.SetValue(4000*4000*8,zLocal.GetValue(0)); + outQueueZ.FreeTensor(zLocal); + } +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/CMakeLists.txt new file mode 100644 index 000000000..fd87c7620 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.16) +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_CXX_STANDARD 17) + +set(CCE_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules) +list(APPEND CMAKE_MODULE_PATH ${CCE_CMAKE_PATH}) +project(kernel_samples LANGUAGES CCE CXX) + +add_subdirectory(cmake/cpu) +add_subdirectory(cmake/npu) \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/README.md b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/README.md new file mode 100644 index 000000000..49bee5787 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/README.md @@ -0,0 +1,81 @@ +## 目录结构介绍 +``` +├── AddKernelInvocation +│ ├── cmake // 编译工程文件 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 验证输出数据和真值数据是否一致的验证脚本 +│ │ └── verify_result.py // 真值对比文件 +│ │── add_custom.cpp // 算子kernel实现 +│ │── CMakeLists.txt // 编译工程文件 +│ │── data_utils.h // 数据读入写出函数 +│ │── main.cpp // 主函数，调用算子的应用程序，含CPU域及NPU域调用 +│ └── run.sh // 编译运行算子的脚本 +``` +## 代码实现介绍 +本调用样例中实现的是固定shape为8*2048的Add算子。 +- kernel实现 + Add算子的数学表达式为： + ``` + z = x + y + ``` + 计算逻辑是：Ascend C提供的矢量计算接口的操作元素都为LocalTensor，输入数据需要先搬运进片上存储，然后使用计算接口完成两个输入参数相加，得到最终结果，再搬出到外部存储上。 + + Add算子的实现流程分为3个基本任务：CopyIn，Compute，CopyOut。CopyIn任务负责将Global Memory上的输入Tensor xGm和yGm搬运到Local Memory，分别存储在xLocal、yLocal，Compute任务负责对xLocal、yLocal执行加法操作，计算结果存储在zLocal中，CopyOut任务负责将输出数据从zLocal搬运至Global Memory上的输出Tensor zGm中。具体请参考[add_custom.cpp](./add_custom.cpp)。 + +- 调用实现 + 1. CPU侧运行验证主要通过ICPU_RUN_KF CPU调测宏等CPU调测库提供的接口来完成； + 2. NPU侧运行验证主要通过使用<<<>>>内核调用符来完成。 + + 应用程序通过ASCENDC_CPU_DEBUG 宏区分代码逻辑运行于CPU侧还是NPU侧。 + +## 运行样例算子 + - 打开样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/KernelLaunch/AddKernelInvocation + ``` + - 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ```` + + 配置仿真模式日志文件目录，默认为sim_log。 + ```bash + export CAMODEL_LOG_PATH=./sim_log + ``` + + - 样例执行 + + ```bash + bash run.sh -r [RUN_MODE] -v [SOC_VERSION] + ``` + - RUN_MODE：编译方式，可选择CPU调试，NPU仿真，NPU上板。支持参数为[cpu / sim / npu]，默认值为cpu。 + - SOC_VERSION：昇腾AI处理器型号，如果无法确定具体的[SOC_VERSION]，则在安装昇腾AI处理器的服务器执行npu-smi info命令进行查询，在查询到的“Name”前增加Ascend信息，例如“Name”对应取值为xxxyy，实际配置的[SOC_VERSION]值为Ascendxxxyy。支持以下参数取值（xxx请替换为具体取值）： + - Atlas 推理系列产品（Ascend 310P处理器）参数值：Ascend310P1、Ascend310P3 + - Atlas 训练系列产品参数值：AscendxxxA、AscendxxxB + - Atlas A2训练系列产品参数值：AscendxxxB1、AscendxxxB2、AscendxxxB3、AscendxxxB4 + + 注：针对Atlas 训练系列产品使用NPU仿真调试，会存在精度问题，可选择其他芯片进行NPU仿真调试。 + + 示例如下。 + ```bash + bash run.sh -r cpu -v Ascend310P1 + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2023/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp new file mode 100644 index 000000000..d2f17a227 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * + * Function : z = x + y + * This sample is a very basic sample that implements vector add on Ascend plaform. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + xGm.SetGlobalBuffer((__gm__ half*)x , 2048); + yGm.SetGlobalBuffer((__gm__ half*)y , 2048); + zGm.SetGlobalBuffer((__gm__ half*)z ,2048); + pipe.InitBuffer(inQueueX, BUFFER_NUM, 4000 * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, 4000 * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, 4000 * sizeof(half)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < 1; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * 2048], 2048); + DataCopy(yLocal, yGm[progress * 2048], 2048); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, 2048); + + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + + DataCopy(zGm, zLocal, 2048); + zGm.SetValue(4000*4000*8,zLocal.GetValue(0)); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + KernelAdd op; + op.Init(x, y, z, 16384, 8); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce new file mode 100644 index 000000000..6bdd65bd9 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce @@ -0,0 +1,117 @@ +extern "C" __global__ [aicore] void add_custom(__gm__ uint8_t* __restrict__ arg0, __gm__ uint8_t* __restrict__ arg1, __gm__ uint8_t* __restrict__ arg2) { +set_ffts_base_addr((uint64_t)0xffffa8a896a0); +set_atomic_none(); +set_padding((uint64_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)1); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)2); +copy_gm_to_ubuf((__ubuf__ void*)0xffffa0844010, (__gm__ void*)arg0, (uint8_t)0, (uint16_t)1, (uint16_t)128, (uint16_t)0, (uint16_t)0); +copy_gm_to_ubuf((__ubuf__ void*)0xffffa0847e90, (__gm__ void*)arg1, (uint8_t)0, (uint16_t)1, (uint16_t)128, (uint16_t)0, (uint16_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)1); +wait_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)0); +#error "sbitset0 NO PROCESS RETURN VALUE" +wait_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)1); +#error "sbitset0 NO PROCESS RETURN VALUE" +plt_b16((uint32_t &)2048, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)0, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)0, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)1, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)0, (Literal)4, (vector_bool)1); +plt_b16((uint32_t &)1920, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)128, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)128, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)2, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)128, (Literal)4, (vector_bool)2); +plt_b16((uint32_t &)1792, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)256, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)256, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)3, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)256, (Literal)4, (vector_bool)3); +plt_b16((uint32_t &)1664, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)384, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)384, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)4, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)384, (Literal)4, (vector_bool)4); +plt_b16((uint32_t &)1536, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)512, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)512, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)5, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)512, (Literal)4, (vector_bool)5); +plt_b16((uint32_t &)1408, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)640, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)640, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)6, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)640, (Literal)4, (vector_bool)6); +plt_b16((uint32_t &)1280, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)768, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)768, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)7, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)768, (Literal)4, (vector_bool)7); +plt_b16((uint32_t &)1152, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)896, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)896, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)8, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)896, (Literal)4, (vector_bool)8); +plt_b16((uint32_t &)1024, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1024, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1024, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)9, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1024, (Literal)4, (vector_bool)9); +plt_b16((uint32_t &)896, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1152, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1152, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)10, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1152, (Literal)4, (vector_bool)10); +plt_b16((uint32_t &)768, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1280, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1280, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)11, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1280, (Literal)4, (vector_bool)11); +plt_b16((uint32_t &)640, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1408, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1408, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)12, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1408, (Literal)4, (vector_bool)12); +plt_b16((uint32_t &)512, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1536, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1536, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)13, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1536, (Literal)4, (vector_bool)13); +plt_b16((uint32_t &)384, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1664, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1664, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)14, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1664, (Literal)4, (vector_bool)14); +plt_b16((uint32_t &)256, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1792, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1792, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)15, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1792, (Literal)4, (vector_bool)15); +plt_b16((uint32_t &)128, (Literal)1); +vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1920, (Literal)2); +vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1920, (Literal)2); +vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)16, (Literal)0); +vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1920, (Literal)4, (vector_bool)16); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE3, (event_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE2, (event_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE2, (event_t)1); +wait_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE3, (event_t)0); +#error "sbitset0 NO PROCESS RETURN VALUE" +copy_ubuf_to_gm((__gm__ void*)arg2, (__ubuf__ void*)0xffffa084bd10, (uint8_t)0, (uint16_t)1, (uint16_t)128, (uint16_t)0, (uint16_t)0); diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in new file mode 100644 index 000000000..a9b5688ff --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in @@ -0,0 +1,5 @@ +set(CMAKE_CCE_COMPILER "@CMAKE_CCE_COMPILER@") +set(CMAKE_CCE_COMPILER_LOADED 1) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS @CMAKE_CCE_SOURCE_FILE_EXTENSIONS@) +set(CMAKE_CCE_OUTPUT_EXTENSION @CMAKE_CCE_OUTPUT_EXTENSION@) +set(CMAKE_CCE_COMPILER_ENV_VAR "@CMAKE_CCE_COMPILER_ENV_VAR@") diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake new file mode 100644 index 000000000..7d2fc2ddd --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake @@ -0,0 +1,41 @@ +include(CMakeCommonLanguageInclude) + +set(CMAKE_INCLUDE_FLAG_CCE "-I") + +if(UNIX) + set(CMAKE_CCE_OUTPUT_EXTENSION .o) +else() + set(CMAKE_CCE_OUTPUT_EXTENSION .obj) +endif() + +set(_INCLUDED_FILE 0) +set(CMAKE_SHARED_LIBRARY_CCE_FLAGS -fPIC) +set(CMAKE_SHARED_LIBRARY_CREATE_CCE_FLAGS -shared) +set(CMAKE_STATIC_LIBRARY_CREATE_CCE_FLAGS "--cce-build-static-lib") +set(CMAKE_LIBRARY_CREATE_CCE_FLAGS "--cce-fatobj-link") + +if(NOT CMAKE_CCE_COMPILE_OBJECT) + set(CMAKE_CCE_COMPILE_OBJECT + " -xcce ${__IMPLICIT_INCLUDES} ${_CMAKE_CCE_BUILTIN_INCLUDE_PATH} ${_CMAKE_COMPILE_AS_CCE_FLAG} ${_CMAKE_CCE_COMPILE_OPTIONS} ${_CMAKE_CCE_COMMON_COMPILE_OPTIONS} -pthread -o -c ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_LIBRARY) + set(CMAKE_CCE_CREATE_SHARED_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_STATIC_LIBRARY) + set(CMAKE_CCE_CREATE_STATIC_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_MODULE) + set(CMAKE_CCE_CREATE_SHARED_MODULE ${CMAKE_CCE_CREATE_SHARED_LIBRARY}) +endif() + +if(NOT CMAKE_CCE_LINK_EXECUTABLE) + set(CMAKE_CCE_LINK_EXECUTABLE + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ${__IMPLICIT_LINKS}") +endif() + +set(CMAKE_CCE_INFORMATION_LOADED 1) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake new file mode 100755 index 000000000..f3c0a4732 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake @@ -0,0 +1,124 @@ +find_program(CMAKE_CCE_COMPILER NAMES "ccec" PATHS "$ENV{PATH}" DOC "CCE Compiler") + +mark_as_advanced(CMAKE_CCE_COMPILER) + +message(STATUS "CMAKE_CCE_COMPILER: " ${CMAKE_CCE_COMPILER}) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS cce;cpp) +set(CMAKE_CCE_COMPILER_ENV_VAR "CCE") +message(STATUS "CMAKE_CURRENT_LIST_DIR: " ${CMAKE_CURRENT_LIST_DIR}) + +# configure all variables set in this file +configure_file(${CMAKE_CURRENT_LIST_DIR}/CMakeCCECompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeCCECompiler.cmake + @ONLY +) + +message(STATUS "ASCEND_PRODUCT_TYPE:\n" " ${ASCEND_PRODUCT_TYPE}") +message(STATUS "ASCEND_CORE_TYPE:\n" " ${ASCEND_CORE_TYPE}") +message(STATUS "ASCEND_INSTALL_PATH:\n" " ${ASCEND_INSTALL_PATH}") + +if(DEFINED ASCEND_INSTALL_PATH) + set(_CMAKE_ASCEND_INSTALL_PATH ${ASCEND_INSTALL_PATH}) +else() + message(FATAL_ERROR + "no, installation path found, should passing -DASCEND_INSTALL_PATH= in cmake" + ) + set(_CMAKE_ASCEND_INSTALL_PATH) +endif() + + +if(DEFINED ASCEND_PRODUCT_TYPE) + set(_CMAKE_CCE_COMMON_COMPILE_OPTIONS "--cce-auto-sync -mllvm -api-deps-filter") + if(ASCEND_PRODUCT_TYPE STREQUAL "") + message(FATAL_ERROR "ASCEND_PRODUCT_TYPE must be non-empty if set.") + elseif(ASCEND_PRODUCT_TYPE AND NOT ASCEND_PRODUCT_TYPE MATCHES "^Ascend[0-9][0-9][0-9][a-zA-Z]?[1-9]?$") + message(FATAL_ERROR + "ASCEND_PRODUCT_TYPE: ${ASCEND_PRODUCT_TYPE}\n" + "is not one of the following: Ascend910A, Ascend310P1, Ascend910B1, Ascend310B1" + ) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c100") + else() + message(FATAL_ERROR, "only AiCore inside") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310P1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-cube") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-record-overflow=false -mllvm -cce-aicore-addr-transform") + endif() +endif() + +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/runtime/lib64 + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/simulator/${ASCEND_PRODUCT_TYPE}/lib + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/${ASCEND_PRODUCT_TYPE} +) + +# link library +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES stdc++) +if(ASCEND_RUN_MODE STREQUAL "npu") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime) +elseif(ASCEND_RUN_MODE STREQUAL "sim") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES ) + if(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES pem_davinci) + endif() + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime_camodel) +elseif(ASCEND_RUN_MODE STREQUAL "cpu") + message(STATUS "RUN_MODE is cpu") +else() + message(FATAL_ERROR + "ASCEND_RUN_MODE: ${ASCEND_RUN_MODE}\n" + "ASCEND_RUN_MODE must be one of the following: cpu, npu or sim" + ) +endif() +list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES ascendcl) + +set(__IMPLICIT_LINKS) +foreach(dir ${_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES}) + string(APPEND __IMPLICIT_LINKS " -L\"${dir}\"") +endforeach() +foreach(lib ${_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES}) + if(${lib} MATCHES "/") + string(APPEND __IMPLICIT_LINKS " \"${lib}\"") + else() + string(APPEND __IMPLICIT_LINKS " -l${lib}") + endif() +endforeach() + +set(_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/acllib/include + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/include +) +set(__IMPLICIT_INCLUDES) +foreach(inc ${_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES}) + string(APPEND __IMPLICIT_INCLUDES " -I\"${inc}\"") +endforeach() \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake new file mode 100644 index 000000000..f00f227c1 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake @@ -0,0 +1 @@ +set(CMAKE_CCE_COMPILER_WORKS 1 CACHE INTERNAL "") diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/cpu/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/cpu/CMakeLists.txt new file mode 100644 index 000000000..661230b17 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/cpu/CMakeLists.txt @@ -0,0 +1,37 @@ +# cpu +if (NOT DEFINED ENV{CMAKE_PREFIX_PATH}) + set(CMAKE_PREFIX_PATH ${ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/cmake) +endif() + +find_package(tikicpulib REQUIRED) + +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) + +add_executable(${smoke_testcase}_cpu + ${SRC_FILES} +) + +target_include_directories(${smoke_testcase}_cpu PRIVATE + ${ASCEND_INSTALL_PATH}/acllib/include + ${CMAKE_SOURCE_DIR} +) + +target_link_libraries(${smoke_testcase}_cpu PRIVATE + tikicpulib::${ASCEND_PRODUCT_TYPE} + ascendcl +) + +target_compile_options(${smoke_testcase}_cpu PRIVATE + -g +) + +target_compile_definitions(${smoke_testcase}_cpu PRIVATE + _GLIBCXX_USE_CXX11_ABI=0 +) + +set_target_properties(${smoke_testcase}_cpu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/npu/CMakeLists.txt b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/npu/CMakeLists.txt new file mode 100644 index 000000000..fb1786c6b --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cmake/npu/CMakeLists.txt @@ -0,0 +1,26 @@ +# npu +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) +set_source_files_properties(${SRC_FILES} PROPERTIES LANGUAGE CCE) + +add_executable(${smoke_testcase}_npu + ${SRC_FILES} +) + +target_compile_options(${smoke_testcase}_npu PRIVATE + -O2 + -std=c++17 +) + +target_link_directories(${smoke_testcase}_npu PRIVATE + ${ASCEND_INSTALL_PATH}/lib64 +) + +set_target_properties(${smoke_testcase}_npu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) + +add_custom_target(${smoke_testcase}_sim ALL) +add_dependencies(${smoke_testcase}_sim ${smoke_testcase}_npu) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/data_utils.h b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/data_utils.h new file mode 100644 index 000000000..7cdefd866 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/data_utils.h @@ -0,0 +1,203 @@ +/** + * @file data_utils.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef DATA_UTILS_H +#define DATA_UTILS_H +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +typedef enum { + DT_UNDEFINED = -1, + FLOAT = 0, + HALF = 1, + INT8_T = 2, + INT32_T = 3, + UINT8_T = 4, + INT16_T = 6, + UINT16_T = 7, + UINT32_T = 8, + INT64_T = 9, + UINT64_T = 10, + DOUBLE = 11, + BOOL = 12, + STRING = 13, + COMPLEX64 = 16, + COMPLEX128 = 17, + BF16 = 27 +} printDataType; + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stdout, "[ERROR] " fmt "\n", ##args) +#define CHECK_ACL(x) \ + do { \ + aclError __ret = x; \ + if (__ret != ACL_ERROR_NONE) { \ + std::cerr << __FILE__ << ":" << __LINE__ << " aclError:" << __ret << std::endl; \ + } \ + } while (0); + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t &fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file"); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintHalfData(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(6) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, printDataType dataType, size_t elementsPerRow = 16) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case HALF: + DoPrintHalfData(reinterpret_cast(data), count, elementsPerRow); + break; + case FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } + std::cout << std::endl; +} +#endif // DATA_UTILS_H diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp new file mode 100644 index 000000000..bd760f1c6 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp @@ -0,0 +1,82 @@ +/** + * @file main.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "data_utils.h" +#ifndef ASCENDC_CPU_DEBUG +#include "acl/acl.h" +extern void add_custom_do(uint32_t coreDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z); +#else +#include "tikicpulib.h" +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z); +#endif + +int32_t main(int32_t argc, char *argv[]) +{ + uint32_t blockDim = 1; + size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + +#ifdef ASCENDC_CPU_DEBUG + uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *y = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *z = (uint8_t *)AscendC::GmAlloc(outputByteSize); + + ReadFile("./input/input_x.bin", inputByteSize, x, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, y, inputByteSize); + + AscendC::SetKernelMode(KernelMode::AIV_MODE); + ICPU_RUN_KF(add_custom, blockDim, x, y, z); // use this macro for cpu debug + + WriteFile("./output/output_z.bin", z, outputByteSize); + + AscendC::GmFree((void *)x); + AscendC::GmFree((void *)y); + AscendC::GmFree((void *)z); +#else + CHECK_ACL(aclInit("./scripts/acl.json")); + int32_t deviceId = 0; + CHECK_ACL(aclrtSetDevice(deviceId)); + aclrtStream stream = nullptr; + CHECK_ACL(aclrtCreateStream(&stream)); + + uint8_t *xHost, *yHost, *zHost; + uint8_t *xDevice, *yDevice, *zDevice; + + CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&zHost), outputByteSize)); + CHECK_ACL(aclrtMalloc((void **)&xDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&yDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&zDevice, outputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input/input_x.bin", inputByteSize, xHost, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, yHost, inputByteSize); + + CHECK_ACL(aclrtMemcpy(xDevice, inputByteSize, xHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + CHECK_ACL(aclrtMemcpy(yDevice, inputByteSize, yHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + + add_custom_do(blockDim, nullptr, stream, xDevice, yDevice, zDevice); + CHECK_ACL(aclrtSynchronizeStream(stream)); + + CHECK_ACL(aclrtMemcpy(zHost, outputByteSize, zDevice, outputByteSize, ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output/output_z.bin", zHost, outputByteSize); + + CHECK_ACL(aclrtFree(xDevice)); + CHECK_ACL(aclrtFree(yDevice)); + CHECK_ACL(aclrtFree(zDevice)); + CHECK_ACL(aclrtFreeHost(xHost)); + CHECK_ACL(aclrtFreeHost(yHost)); + CHECK_ACL(aclrtFreeHost(zHost)); + + CHECK_ACL(aclrtDestroyStream(stream)); + CHECK_ACL(aclrtResetDevice(deviceId)); + CHECK_ACL(aclFinalize()); +#endif + return 0; +} diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/run.sh b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/run.sh new file mode 100644 index 000000000..e741e1224 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/run.sh @@ -0,0 +1,130 @@ +#!/bin/bash +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +declare -A VersionMap +VersionMap["Ascend910A"]="Ascend910A" +VersionMap["Ascend910B"]="Ascend910A" +VersionMap["Ascend910ProA"]="Ascend910A" +VersionMap["Ascend910ProB"]="Ascend910A" +VersionMap["Ascend910PremiumA"]="Ascend910A" +VersionMap["Ascend310B1"]="Ascend310B1" +VersionMap["Ascend310B2"]="Ascend310B1" +VersionMap["Ascend310B3"]="Ascend310B1" +VersionMap["Ascend310B4"]="Ascend310B1" +VersionMap["Ascend310P1"]="Ascend310P1" +VersionMap["Ascend310P3"]="Ascend310P1" +VersionMap["Ascend910B1"]="Ascend910B1" +VersionMap["Ascend910B2"]="Ascend910B1" +VersionMap["Ascend910B3"]="Ascend910B1" +VersionMap["Ascend910B4"]="Ascend910B1" +# legacy +VersionMap["ascend910"]="Ascend910A" +VersionMap["ascend310p"]="Ascend310P1" +VersionMap["ascend310B1"]="Ascend310B1" +VersionMap["ascend910B1"]="Ascend910B1" + +FILE_NAME="add" + +SHORT=r:,v:,i:, +LONG=run-mode:,soc-version:,install-path:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" + +while :; do + case "$1" in + -r | --run-mode) + RUN_MODE="$2" + shift 2 + ;; + -v | --soc-version) + SOC_VERSION="$2" + shift 2 + ;; + -i | --install-path) + ASCEND_INSTALL_PATH="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +# in case of running op in simulator, use stub so instead +if [ "${RUN_MODE}" = "sim" ]; then + export LD_LIBRARY_PATH=$_ASCEND_INSTALL_PATH/runtime/lib64/stub:$LD_LIBRARY_PATH + if [ ! $CAMODEL_LOG_PATH ]; then + export CAMODEL_LOG_PATH=./sim_log + fi + rm -rf $CAMODEL_LOG_PATH + mkdir -p $CAMODEL_LOG_PATH +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash + +if [[ " ${!VersionMap[*]} " != *" $SOC_VERSION "* ]]; then + echo "ERROR: SOC_VERSION should be in [${!VersionMap[*]}]" + exit -1 +fi +_SOC_VERSION=${VersionMap[$SOC_VERSION]} + +if [ $_SOC_VERSION"x" = "Ascend910Ax" ] || [ $_SOC_VERSION"x" = "Ascend310P1x" ] || [ $_SOC_VERSION"x" = "Ascend310B1x" ]; then + CORE_TYPE="AiCore" +elif [ $_SOC_VERSION"x" = "Ascend910B1x" ]; then + CORE_TYPE="VectorCore" +fi + +RUN_MODE_LIST="cpu sim npu" +if [[ " $RUN_MODE_LIST " != *" $RUN_MODE "* ]]; then + echo "ERROR: RUN_MODE error, This sample only support specify cpu, sim or npu!" + exit -1 +fi + +set -e +rm -rf build *_cpu *_sim *_npu cceprint npuchk *log *.vcd + +rm -rf build +mkdir -p build +cmake -B build \ + -Dsmoke_testcase=${FILE_NAME} \ + -DASCEND_PRODUCT_TYPE=${_SOC_VERSION} \ + -DASCEND_CORE_TYPE=${CORE_TYPE} \ + -DASCEND_RUN_MODE=${RUN_MODE} \ + -DASCEND_INSTALL_PATH=${_ASCEND_INSTALL_PATH} +cmake --build build --target ${FILE_NAME}_${RUN_MODE} +if [ $? -ne 0 ]; then + echo "ERROR: compile op on failed!" + exit -1 +fi +echo "INFO: compile op on ${RUN_MODE} succeed!" + +rm -rf input/*.bin output/*.bin +python3 scripts/gen_data.py +(export LD_LIBRARY_PATH=${_ASCEND_INSTALL_PATH}/tools/simulator/${_SOC_VERSION}/lib:$LD_LIBRARY_PATH && ./${FILE_NAME}_${RUN_MODE}) +if [ $? -ne 0 ]; then + echo "ERROR: execute op on ${RUN_MODE} failed!" + exit -1 +fi +echo "INFO: execute op on ${RUN_MODE} succeed!" +python3 scripts/verify_result.py output/output_z.bin output/golden.bin + +rm -rf *log *.vcd diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/acl.json b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/gen_data.py new file mode 100644 index 000000000..a85a84a15 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/gen_data.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/verify_result.py new file mode 100644 index 000000000..1a21d809a --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 100: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/InvalidMemory/README.md b/operator_contrib/CommonMistake/InvalidMemory/README.md new file mode 100644 index 000000000..f1c3b0667 --- /dev/null +++ b/operator_contrib/CommonMistake/InvalidMemory/README.md @@ -0,0 +1,51 @@ +## 样例说明 + +该样例用于复现算子Gm以及UB越界问题 + +## 测试说明 + +编译部署AddCustom后，运行AclNNlnvocation调用，即可复现错误，界面会输出 + +```c++ +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Copy input[1] success +[INFO] Create stream success +[INFO] Execute aclnnAddCustomGetWorkspaceSize success, workspace size 0 +[INFO] Execute aclnnAddCustom success +[ERROR] Synchronize stream failed. error code is 507015 +[ERROR] Run op failed +[INFO] Reset Device success +[INFO] Destory resource success +ERROR: acl executable run failed! please check your project! +``` + +状态码：507015->aicore执行异常。 + + +1.在/var/log/npu/slog使用grep -rn "error"，可以检索到以下代码: + +```c +debug/device-app-1111824/device-app-1111824_20240816162800144.log:4287:[ERROR] RUNTIME(1111824,execute_add_op):2024-08-16-16:28:00.311.109 [device_error_proc.cc:1230]1111824 ProcessStarsCoreErrorInfo:The extend info: errcode:(0, 0x8000, 0) errorStr: When the D-cache reads and writes data to the UB, the response value returned by the bus is a non-zero value. fixp_error0 info: 0x1044, fixp_error1 info: 0x5d fsmId:0, tslot:0, thread:0, ctxid:0, blk:0, sublk:0, subErrType:4. + +``` + +## 解决思路 +1、阅读代码，检查API使用是否满足通用对齐约束 +2、通过添加printf打印，直接在输出各个API的地址偏移（需要产品支持printf调试API） +3、构造相同输入，通过孪生调试功能的CPU侧调试进行定位，该方式可以直接显示错误原因及相关代码行，本例会输出类似： + + +[ERROR][CORE_0][pid 2909419] error happened! ========= +SIGSEGV Signal **(Invalid memory reference)** catched, backtrace info: +[#0] 0x000000000041d608: Handler(int) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/lib/include/kern_fwk.h:106 +[#1] 0x00000000004118f8: AscendC::GlobalTensor::**SetValue** (unsigned long, float16::Fp16T) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/kernel_tensor_impl.h:626 +[#2] 0x000000000040ea44: KernelAdd::CopyOut(int) at /root/libotao/ascend-error-example/Test_memeory_error/AddKernelInvocation/**add_custom.cpp:68** (discriminator 3) +[#3] 0x000000000040e7a0: KernelAdd::Process() at /root/libotao/ascend-error-example/Test_memeory_error/AddKernelInvocation/add_custom.cpp:34 (discriminator 2) +[#4] 0x000000000040ba14: add_custom at /root/libotao/ascend-error-example/Test_memeory_error/AddKernelInvocation/add_custom.cpp:85 +[#5] 0x000000000041ce44: main at /root/libotao/ascend-error-example/Test_memeory_error/AddKernelInvocation/main.cpp:34 (discriminator 127) +[#6] 0x000000000040b8f0: _start at ??:? + diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/README.md b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/README.md new file mode 100644 index 000000000..b8167d88f --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/README.md @@ -0,0 +1,54 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ └── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ```cpp + // 获取算子使用的workspace空间大小 + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + // 执行算子 + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + 以命令行方式下载样例代码，master分支为例。 + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ```bash + bash run.sh + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2024/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..11bb4aeca --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** + * @file common.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..66b6ea2c3 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,180 @@ +/** + * @file op_runner.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "acl/acl.h" +#include "aclnn/acl_meta.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..6d8ee0905 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** + * @file operator_desc.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/run.sh new file mode 100644 index 000000000..e1cec0584 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while :; do + case "$1" in + # float16, float, int32 + -v | --dtype) + DTYPE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash +export DDK_PATH=$_ASCEND_INSTALL_PATH +export NPU_HOST_LIB=$_ASCEND_INSTALL_PATH/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR + rm -rf build + mkdir -p build + cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + ./execute_add_op + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output_z.bin output/golden.bin + if [ $? -ne 0 ]; then + echo "ERROR: verify result failed!" + return 1 + fi +} + +main diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..6d15edfe0 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..1a21d809a --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 100: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..992759c95 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/common.cpp @@ -0,0 +1,80 @@ +/** + * @file common.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "common.h" + +#include +#include +#include + +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + size_t writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..481a31df8 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/main.cpp @@ -0,0 +1,162 @@ +/** + * @file main.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include +#include +#include + +#include +#include + +#include "acl/acl.h" +#include "common.h" +#include "op_runner.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..7e1623aed --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,453 @@ +/** + * @file op_runner.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "op_runner.h" + +#include +#include + +#include "acl/acl_op_compiler.h" +#include "aclnn_add_custom.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = + aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), nullptr, 0, + GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = + aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), nullptr, 0, + GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = + aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..da04cf6c9 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,51 @@ +/** + * @file operator_desc.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "operator_desc.h" + +#include "common.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/build.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..3a6671209 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,73 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 8; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..323f3076f --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,22 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..1e6a83fba --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,96 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); + // pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, + uint8_t *workspace, uint8_t *tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/CMakeLists.txt new file mode 100644 index 000000000..fd87c7620 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.16) +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_CXX_STANDARD 17) + +set(CCE_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules) +list(APPEND CMAKE_MODULE_PATH ${CCE_CMAKE_PATH}) +project(kernel_samples LANGUAGES CCE CXX) + +add_subdirectory(cmake/cpu) +add_subdirectory(cmake/npu) \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/README.md b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/README.md new file mode 100644 index 000000000..49bee5787 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/README.md @@ -0,0 +1,81 @@ +## 目录结构介绍 +``` +├── AddKernelInvocation +│ ├── cmake // 编译工程文件 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 验证输出数据和真值数据是否一致的验证脚本 +│ │ └── verify_result.py // 真值对比文件 +│ │── add_custom.cpp // 算子kernel实现 +│ │── CMakeLists.txt // 编译工程文件 +│ │── data_utils.h // 数据读入写出函数 +│ │── main.cpp // 主函数，调用算子的应用程序，含CPU域及NPU域调用 +│ └── run.sh // 编译运行算子的脚本 +``` +## 代码实现介绍 +本调用样例中实现的是固定shape为8*2048的Add算子。 +- kernel实现 + Add算子的数学表达式为： + ``` + z = x + y + ``` + 计算逻辑是：Ascend C提供的矢量计算接口的操作元素都为LocalTensor，输入数据需要先搬运进片上存储，然后使用计算接口完成两个输入参数相加，得到最终结果，再搬出到外部存储上。 + + Add算子的实现流程分为3个基本任务：CopyIn，Compute，CopyOut。CopyIn任务负责将Global Memory上的输入Tensor xGm和yGm搬运到Local Memory，分别存储在xLocal、yLocal，Compute任务负责对xLocal、yLocal执行加法操作，计算结果存储在zLocal中，CopyOut任务负责将输出数据从zLocal搬运至Global Memory上的输出Tensor zGm中。具体请参考[add_custom.cpp](./add_custom.cpp)。 + +- 调用实现 + 1. CPU侧运行验证主要通过ICPU_RUN_KF CPU调测宏等CPU调测库提供的接口来完成； + 2. NPU侧运行验证主要通过使用<<<>>>内核调用符来完成。 + + 应用程序通过ASCENDC_CPU_DEBUG 宏区分代码逻辑运行于CPU侧还是NPU侧。 + +## 运行样例算子 + - 打开样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/KernelLaunch/AddKernelInvocation + ``` + - 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ```` + + 配置仿真模式日志文件目录，默认为sim_log。 + ```bash + export CAMODEL_LOG_PATH=./sim_log + ``` + + - 样例执行 + + ```bash + bash run.sh -r [RUN_MODE] -v [SOC_VERSION] + ``` + - RUN_MODE：编译方式，可选择CPU调试，NPU仿真，NPU上板。支持参数为[cpu / sim / npu]，默认值为cpu。 + - SOC_VERSION：昇腾AI处理器型号，如果无法确定具体的[SOC_VERSION]，则在安装昇腾AI处理器的服务器执行npu-smi info命令进行查询，在查询到的“Name”前增加Ascend信息，例如“Name”对应取值为xxxyy，实际配置的[SOC_VERSION]值为Ascendxxxyy。支持以下参数取值（xxx请替换为具体取值）： + - Atlas 推理系列产品（Ascend 310P处理器）参数值：Ascend310P1、Ascend310P3 + - Atlas 训练系列产品参数值：AscendxxxA、AscendxxxB + - Atlas A2训练系列产品参数值：AscendxxxB1、AscendxxxB2、AscendxxxB3、AscendxxxB4 + + 注：针对Atlas 训练系列产品使用NPU仿真调试，会存在精度问题，可选择其他芯片进行NPU仿真调试。 + + 示例如下。 + ```bash + bash run.sh -r cpu -v Ascend310P1 + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2023/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp new file mode 100644 index 000000000..4efceda1a --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp @@ -0,0 +1,91 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; + +constexpr int32_t TOTAL_LENGTH = 8 * 2048; // total length of data +constexpr int32_t USE_CORE_NUM = 1; // num of core used +constexpr int32_t BLOCK_LENGTH = TOTAL_LENGTH / USE_CORE_NUM; // length computed of each core +constexpr int32_t TILE_NUM = 8; // split data into 8 tiles for each core +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue +constexpr int32_t TILE_LENGTH = BLOCK_LENGTH / TILE_NUM / BUFFER_NUM; // seperate to 2 parts, due to double buffer + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z) + { + xGm.SetGlobalBuffer((__gm__ half *)x + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + yGm.SetGlobalBuffer((__gm__ half *)y + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + zGm.SetGlobalBuffer((__gm__ half *)z + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + // pipe.InitBuffer(inQueueX, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + } + __aicore__ inline void Process() + { + int32_t loopCount = TILE_NUM * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * TILE_LENGTH], TILE_LENGTH); + DataCopy(yLocal, yGm[progress * TILE_LENGTH], TILE_LENGTH); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, TILE_LENGTH); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * TILE_LENGTH], zLocal, TILE_LENGTH); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +{ + KernelAdd op; + op.Init(x, y, z); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +{ + add_custom<<>>(x, y, z); +} +#endif diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce new file mode 100644 index 000000000..2c5d04cb7 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce @@ -0,0 +1,15 @@ +extern "C" __global__ [aicore] void add_custom(__gm__ uint8_t* __restrict__ arg0, __gm__ uint8_t* __restrict__ arg1, __gm__ uint8_t* __restrict__ arg2) { +set_ffts_base_addr((uint64_t)0xffff931e96a0); +set_atomic_none(); +set_padding((uint64_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)0); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)1); +#error "sff0 NO PROCESS RETURN VALUE" +#error "sbitset1 NO PROCESS RETURN VALUE" +set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)2); +copy_gm_to_ubuf((__ubuf__ void*)0xffff8afa4010, (__gm__ void*)arg0, (uint8_t)0, (uint16_t)1, (uint16_t)64, (uint16_t)0, (uint16_t)0); +copy_gm_to_ubuf((__ubuf__ void*)0xffff8afa5010, (__gm__ void*)arg1, (uint8_t)0, (uint16_t)1, (uint16_t)64, (uint16_t)0, (uint16_t)0); diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in new file mode 100644 index 000000000..a9b5688ff --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in @@ -0,0 +1,5 @@ +set(CMAKE_CCE_COMPILER "@CMAKE_CCE_COMPILER@") +set(CMAKE_CCE_COMPILER_LOADED 1) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS @CMAKE_CCE_SOURCE_FILE_EXTENSIONS@) +set(CMAKE_CCE_OUTPUT_EXTENSION @CMAKE_CCE_OUTPUT_EXTENSION@) +set(CMAKE_CCE_COMPILER_ENV_VAR "@CMAKE_CCE_COMPILER_ENV_VAR@") diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake new file mode 100644 index 000000000..7d2fc2ddd --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake @@ -0,0 +1,41 @@ +include(CMakeCommonLanguageInclude) + +set(CMAKE_INCLUDE_FLAG_CCE "-I") + +if(UNIX) + set(CMAKE_CCE_OUTPUT_EXTENSION .o) +else() + set(CMAKE_CCE_OUTPUT_EXTENSION .obj) +endif() + +set(_INCLUDED_FILE 0) +set(CMAKE_SHARED_LIBRARY_CCE_FLAGS -fPIC) +set(CMAKE_SHARED_LIBRARY_CREATE_CCE_FLAGS -shared) +set(CMAKE_STATIC_LIBRARY_CREATE_CCE_FLAGS "--cce-build-static-lib") +set(CMAKE_LIBRARY_CREATE_CCE_FLAGS "--cce-fatobj-link") + +if(NOT CMAKE_CCE_COMPILE_OBJECT) + set(CMAKE_CCE_COMPILE_OBJECT + " -xcce ${__IMPLICIT_INCLUDES} ${_CMAKE_CCE_BUILTIN_INCLUDE_PATH} ${_CMAKE_COMPILE_AS_CCE_FLAG} ${_CMAKE_CCE_COMPILE_OPTIONS} ${_CMAKE_CCE_COMMON_COMPILE_OPTIONS} -pthread -o -c ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_LIBRARY) + set(CMAKE_CCE_CREATE_SHARED_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_STATIC_LIBRARY) + set(CMAKE_CCE_CREATE_STATIC_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_MODULE) + set(CMAKE_CCE_CREATE_SHARED_MODULE ${CMAKE_CCE_CREATE_SHARED_LIBRARY}) +endif() + +if(NOT CMAKE_CCE_LINK_EXECUTABLE) + set(CMAKE_CCE_LINK_EXECUTABLE + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ${__IMPLICIT_LINKS}") +endif() + +set(CMAKE_CCE_INFORMATION_LOADED 1) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake new file mode 100755 index 000000000..f3c0a4732 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake @@ -0,0 +1,124 @@ +find_program(CMAKE_CCE_COMPILER NAMES "ccec" PATHS "$ENV{PATH}" DOC "CCE Compiler") + +mark_as_advanced(CMAKE_CCE_COMPILER) + +message(STATUS "CMAKE_CCE_COMPILER: " ${CMAKE_CCE_COMPILER}) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS cce;cpp) +set(CMAKE_CCE_COMPILER_ENV_VAR "CCE") +message(STATUS "CMAKE_CURRENT_LIST_DIR: " ${CMAKE_CURRENT_LIST_DIR}) + +# configure all variables set in this file +configure_file(${CMAKE_CURRENT_LIST_DIR}/CMakeCCECompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeCCECompiler.cmake + @ONLY +) + +message(STATUS "ASCEND_PRODUCT_TYPE:\n" " ${ASCEND_PRODUCT_TYPE}") +message(STATUS "ASCEND_CORE_TYPE:\n" " ${ASCEND_CORE_TYPE}") +message(STATUS "ASCEND_INSTALL_PATH:\n" " ${ASCEND_INSTALL_PATH}") + +if(DEFINED ASCEND_INSTALL_PATH) + set(_CMAKE_ASCEND_INSTALL_PATH ${ASCEND_INSTALL_PATH}) +else() + message(FATAL_ERROR + "no, installation path found, should passing -DASCEND_INSTALL_PATH= in cmake" + ) + set(_CMAKE_ASCEND_INSTALL_PATH) +endif() + + +if(DEFINED ASCEND_PRODUCT_TYPE) + set(_CMAKE_CCE_COMMON_COMPILE_OPTIONS "--cce-auto-sync -mllvm -api-deps-filter") + if(ASCEND_PRODUCT_TYPE STREQUAL "") + message(FATAL_ERROR "ASCEND_PRODUCT_TYPE must be non-empty if set.") + elseif(ASCEND_PRODUCT_TYPE AND NOT ASCEND_PRODUCT_TYPE MATCHES "^Ascend[0-9][0-9][0-9][a-zA-Z]?[1-9]?$") + message(FATAL_ERROR + "ASCEND_PRODUCT_TYPE: ${ASCEND_PRODUCT_TYPE}\n" + "is not one of the following: Ascend910A, Ascend310P1, Ascend910B1, Ascend310B1" + ) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c100") + else() + message(FATAL_ERROR, "only AiCore inside") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310P1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-cube") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-record-overflow=false -mllvm -cce-aicore-addr-transform") + endif() +endif() + +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/runtime/lib64 + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/simulator/${ASCEND_PRODUCT_TYPE}/lib + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/${ASCEND_PRODUCT_TYPE} +) + +# link library +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES stdc++) +if(ASCEND_RUN_MODE STREQUAL "npu") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime) +elseif(ASCEND_RUN_MODE STREQUAL "sim") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES ) + if(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES pem_davinci) + endif() + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime_camodel) +elseif(ASCEND_RUN_MODE STREQUAL "cpu") + message(STATUS "RUN_MODE is cpu") +else() + message(FATAL_ERROR + "ASCEND_RUN_MODE: ${ASCEND_RUN_MODE}\n" + "ASCEND_RUN_MODE must be one of the following: cpu, npu or sim" + ) +endif() +list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES ascendcl) + +set(__IMPLICIT_LINKS) +foreach(dir ${_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES}) + string(APPEND __IMPLICIT_LINKS " -L\"${dir}\"") +endforeach() +foreach(lib ${_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES}) + if(${lib} MATCHES "/") + string(APPEND __IMPLICIT_LINKS " \"${lib}\"") + else() + string(APPEND __IMPLICIT_LINKS " -l${lib}") + endif() +endforeach() + +set(_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/acllib/include + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/include +) +set(__IMPLICIT_INCLUDES) +foreach(inc ${_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES}) + string(APPEND __IMPLICIT_INCLUDES " -I\"${inc}\"") +endforeach() \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake new file mode 100644 index 000000000..f00f227c1 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake @@ -0,0 +1 @@ +set(CMAKE_CCE_COMPILER_WORKS 1 CACHE INTERNAL "") diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/cpu/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/cpu/CMakeLists.txt new file mode 100644 index 000000000..661230b17 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/cpu/CMakeLists.txt @@ -0,0 +1,37 @@ +# cpu +if (NOT DEFINED ENV{CMAKE_PREFIX_PATH}) + set(CMAKE_PREFIX_PATH ${ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/cmake) +endif() + +find_package(tikicpulib REQUIRED) + +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) + +add_executable(${smoke_testcase}_cpu + ${SRC_FILES} +) + +target_include_directories(${smoke_testcase}_cpu PRIVATE + ${ASCEND_INSTALL_PATH}/acllib/include + ${CMAKE_SOURCE_DIR} +) + +target_link_libraries(${smoke_testcase}_cpu PRIVATE + tikicpulib::${ASCEND_PRODUCT_TYPE} + ascendcl +) + +target_compile_options(${smoke_testcase}_cpu PRIVATE + -g +) + +target_compile_definitions(${smoke_testcase}_cpu PRIVATE + _GLIBCXX_USE_CXX11_ABI=0 +) + +set_target_properties(${smoke_testcase}_cpu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/npu/CMakeLists.txt b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/npu/CMakeLists.txt new file mode 100644 index 000000000..fb1786c6b --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cmake/npu/CMakeLists.txt @@ -0,0 +1,26 @@ +# npu +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) +set_source_files_properties(${SRC_FILES} PROPERTIES LANGUAGE CCE) + +add_executable(${smoke_testcase}_npu + ${SRC_FILES} +) + +target_compile_options(${smoke_testcase}_npu PRIVATE + -O2 + -std=c++17 +) + +target_link_directories(${smoke_testcase}_npu PRIVATE + ${ASCEND_INSTALL_PATH}/lib64 +) + +set_target_properties(${smoke_testcase}_npu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) + +add_custom_target(${smoke_testcase}_sim ALL) +add_dependencies(${smoke_testcase}_sim ${smoke_testcase}_npu) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/data_utils.h b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/data_utils.h new file mode 100644 index 000000000..7cdefd866 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/data_utils.h @@ -0,0 +1,203 @@ +/** + * @file data_utils.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef DATA_UTILS_H +#define DATA_UTILS_H +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +typedef enum { + DT_UNDEFINED = -1, + FLOAT = 0, + HALF = 1, + INT8_T = 2, + INT32_T = 3, + UINT8_T = 4, + INT16_T = 6, + UINT16_T = 7, + UINT32_T = 8, + INT64_T = 9, + UINT64_T = 10, + DOUBLE = 11, + BOOL = 12, + STRING = 13, + COMPLEX64 = 16, + COMPLEX128 = 17, + BF16 = 27 +} printDataType; + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stdout, "[ERROR] " fmt "\n", ##args) +#define CHECK_ACL(x) \ + do { \ + aclError __ret = x; \ + if (__ret != ACL_ERROR_NONE) { \ + std::cerr << __FILE__ << ":" << __LINE__ << " aclError:" << __ret << std::endl; \ + } \ + } while (0); + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t &fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file"); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintHalfData(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(6) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, printDataType dataType, size_t elementsPerRow = 16) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case HALF: + DoPrintHalfData(reinterpret_cast(data), count, elementsPerRow); + break; + case FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } + std::cout << std::endl; +} +#endif // DATA_UTILS_H diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp new file mode 100644 index 000000000..bd760f1c6 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp @@ -0,0 +1,82 @@ +/** + * @file main.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "data_utils.h" +#ifndef ASCENDC_CPU_DEBUG +#include "acl/acl.h" +extern void add_custom_do(uint32_t coreDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z); +#else +#include "tikicpulib.h" +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z); +#endif + +int32_t main(int32_t argc, char *argv[]) +{ + uint32_t blockDim = 1; + size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + +#ifdef ASCENDC_CPU_DEBUG + uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *y = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *z = (uint8_t *)AscendC::GmAlloc(outputByteSize); + + ReadFile("./input/input_x.bin", inputByteSize, x, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, y, inputByteSize); + + AscendC::SetKernelMode(KernelMode::AIV_MODE); + ICPU_RUN_KF(add_custom, blockDim, x, y, z); // use this macro for cpu debug + + WriteFile("./output/output_z.bin", z, outputByteSize); + + AscendC::GmFree((void *)x); + AscendC::GmFree((void *)y); + AscendC::GmFree((void *)z); +#else + CHECK_ACL(aclInit("./scripts/acl.json")); + int32_t deviceId = 0; + CHECK_ACL(aclrtSetDevice(deviceId)); + aclrtStream stream = nullptr; + CHECK_ACL(aclrtCreateStream(&stream)); + + uint8_t *xHost, *yHost, *zHost; + uint8_t *xDevice, *yDevice, *zDevice; + + CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&zHost), outputByteSize)); + CHECK_ACL(aclrtMalloc((void **)&xDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&yDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&zDevice, outputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input/input_x.bin", inputByteSize, xHost, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, yHost, inputByteSize); + + CHECK_ACL(aclrtMemcpy(xDevice, inputByteSize, xHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + CHECK_ACL(aclrtMemcpy(yDevice, inputByteSize, yHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + + add_custom_do(blockDim, nullptr, stream, xDevice, yDevice, zDevice); + CHECK_ACL(aclrtSynchronizeStream(stream)); + + CHECK_ACL(aclrtMemcpy(zHost, outputByteSize, zDevice, outputByteSize, ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output/output_z.bin", zHost, outputByteSize); + + CHECK_ACL(aclrtFree(xDevice)); + CHECK_ACL(aclrtFree(yDevice)); + CHECK_ACL(aclrtFree(zDevice)); + CHECK_ACL(aclrtFreeHost(xHost)); + CHECK_ACL(aclrtFreeHost(yHost)); + CHECK_ACL(aclrtFreeHost(zHost)); + + CHECK_ACL(aclrtDestroyStream(stream)); + CHECK_ACL(aclrtResetDevice(deviceId)); + CHECK_ACL(aclFinalize()); +#endif + return 0; +} diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/run.sh b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/run.sh new file mode 100644 index 000000000..e741e1224 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/run.sh @@ -0,0 +1,130 @@ +#!/bin/bash +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +declare -A VersionMap +VersionMap["Ascend910A"]="Ascend910A" +VersionMap["Ascend910B"]="Ascend910A" +VersionMap["Ascend910ProA"]="Ascend910A" +VersionMap["Ascend910ProB"]="Ascend910A" +VersionMap["Ascend910PremiumA"]="Ascend910A" +VersionMap["Ascend310B1"]="Ascend310B1" +VersionMap["Ascend310B2"]="Ascend310B1" +VersionMap["Ascend310B3"]="Ascend310B1" +VersionMap["Ascend310B4"]="Ascend310B1" +VersionMap["Ascend310P1"]="Ascend310P1" +VersionMap["Ascend310P3"]="Ascend310P1" +VersionMap["Ascend910B1"]="Ascend910B1" +VersionMap["Ascend910B2"]="Ascend910B1" +VersionMap["Ascend910B3"]="Ascend910B1" +VersionMap["Ascend910B4"]="Ascend910B1" +# legacy +VersionMap["ascend910"]="Ascend910A" +VersionMap["ascend310p"]="Ascend310P1" +VersionMap["ascend310B1"]="Ascend310B1" +VersionMap["ascend910B1"]="Ascend910B1" + +FILE_NAME="add" + +SHORT=r:,v:,i:, +LONG=run-mode:,soc-version:,install-path:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" + +while :; do + case "$1" in + -r | --run-mode) + RUN_MODE="$2" + shift 2 + ;; + -v | --soc-version) + SOC_VERSION="$2" + shift 2 + ;; + -i | --install-path) + ASCEND_INSTALL_PATH="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +# in case of running op in simulator, use stub so instead +if [ "${RUN_MODE}" = "sim" ]; then + export LD_LIBRARY_PATH=$_ASCEND_INSTALL_PATH/runtime/lib64/stub:$LD_LIBRARY_PATH + if [ ! $CAMODEL_LOG_PATH ]; then + export CAMODEL_LOG_PATH=./sim_log + fi + rm -rf $CAMODEL_LOG_PATH + mkdir -p $CAMODEL_LOG_PATH +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash + +if [[ " ${!VersionMap[*]} " != *" $SOC_VERSION "* ]]; then + echo "ERROR: SOC_VERSION should be in [${!VersionMap[*]}]" + exit -1 +fi +_SOC_VERSION=${VersionMap[$SOC_VERSION]} + +if [ $_SOC_VERSION"x" = "Ascend910Ax" ] || [ $_SOC_VERSION"x" = "Ascend310P1x" ] || [ $_SOC_VERSION"x" = "Ascend310B1x" ]; then + CORE_TYPE="AiCore" +elif [ $_SOC_VERSION"x" = "Ascend910B1x" ]; then + CORE_TYPE="VectorCore" +fi + +RUN_MODE_LIST="cpu sim npu" +if [[ " $RUN_MODE_LIST " != *" $RUN_MODE "* ]]; then + echo "ERROR: RUN_MODE error, This sample only support specify cpu, sim or npu!" + exit -1 +fi + +set -e +rm -rf build *_cpu *_sim *_npu cceprint npuchk *log *.vcd + +rm -rf build +mkdir -p build +cmake -B build \ + -Dsmoke_testcase=${FILE_NAME} \ + -DASCEND_PRODUCT_TYPE=${_SOC_VERSION} \ + -DASCEND_CORE_TYPE=${CORE_TYPE} \ + -DASCEND_RUN_MODE=${RUN_MODE} \ + -DASCEND_INSTALL_PATH=${_ASCEND_INSTALL_PATH} +cmake --build build --target ${FILE_NAME}_${RUN_MODE} +if [ $? -ne 0 ]; then + echo "ERROR: compile op on failed!" + exit -1 +fi +echo "INFO: compile op on ${RUN_MODE} succeed!" + +rm -rf input/*.bin output/*.bin +python3 scripts/gen_data.py +(export LD_LIBRARY_PATH=${_ASCEND_INSTALL_PATH}/tools/simulator/${_SOC_VERSION}/lib:$LD_LIBRARY_PATH && ./${FILE_NAME}_${RUN_MODE}) +if [ $? -ne 0 ]; then + echo "ERROR: execute op on ${RUN_MODE} failed!" + exit -1 +fi +echo "INFO: execute op on ${RUN_MODE} succeed!" +python3 scripts/verify_result.py output/output_z.bin output/golden.bin + +rm -rf *log *.vcd diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/acl.json b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/gen_data.py new file mode 100644 index 000000000..a85a84a15 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/gen_data.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/verify_result.py new file mode 100644 index 000000000..ddb6e4a31 --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 0: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/README.md b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/README.md new file mode 100644 index 000000000..63569de2f --- /dev/null +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/README.md @@ -0,0 +1,63 @@ +## 样例说明 +该样例用于复现aclnn调用算子时，输出 Synchronize stream failed. error code is 507015的错误 + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` +INFO: execute op! +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Copy input[1] success +[INFO] Create stream success +[INFO] Execute GetWorkspaceSize success, workspace size 0 +[INFO] Execute Operator success +[ERROR] Synchronize stream failed. error code is 507015 +[ERROR] Run op failed +[INFO] Reset Device success +[INFO] Destory resource success +ERROR: acl executable run failed! please check your project! + + +``` + +使用grep在plog日志查找“ERROR”关键字有类似信息 +``` +[ERROR] RUNTIME(3397738,execute_op):2024-08-19-10:21:36.550.207 [device_error_proc.cc:1207]3397738 ProcessStarsCoreErrorInfo:report error module_type=5, module_name=EZ9999 +[ERROR] RUNTIME(3397738,execute_op):2024-08-19-10:21:36.550.227 [device_error_proc.cc:1207]3397738 ProcessStarsCoreErrorInfo:The error from device(chipId:0, dieId:0), serial number is 418, there is an aicore error exception, core id is 0, error code = 0, dump info: pc start: 0xe80000025000, current: 0xe80000025b74, vec error info: 0, mte error info: 0x9500000062, ifu error info: 0x5e8000f469c00, ccu error info: 0x4b0000d6, cube error info: 0, biu error info: 0, aic error mask: 0x65000208d000288, para base: 0xe800007e5200. +[INFO] GE(3397738,execute_op):2024-08-19-10:21:36.550.358 [error_manager.cc:355]3397738 ReportInterErrMessage:report error_message, error_code:EZ9999, work_stream_id:339777197738, error_mode:0 +[ERROR] RUNTIME(3397738,execute_op):2024-08-19-10:21:36.550.432 [device_error_proc.cc:1219]3397738 ProcessStarsCoreErrorInfo:report error module_type=5, module_name=EZ9999 +[ERROR] RUNTIME(3397738,execute_op):2024-08-19-10:21:36.550.453 [device_error_proc.cc:1219]3397738 ProcessStarsCoreErrorInfo:The extend info: errcode:(0, 0x200000000000000, 0) errorStr: The MPU address access is invalid. fixp_error0 info: 0x62, fixp_error1 info: 0x95 fsmId:0, tslot:1, thread:0, ctxid:0, blk:0, sublk:0, subErrType:4. +[INFO] GE(3397738,execute_op):2024-08-19-10:21:36.550.497 [error_manager.cc:355]3397738 ReportInterErrMessage:report error_message, error_code:EZ9999, work_stream_id:339777197738, error_mode:0 +[INFO] RUNTIME(3397738,execute_op):2024-08-19-10:21:36.550.529 [device_error_proc.cc:883] 3397738 ProcErrorInfo: finished to process device errInfo, retCode=0. +[DEBUG] RUNTIME(3397738,execute_op):2024-08-19-10:21:36.550.548 [stars_engine.cc:1345] 3397738 ProcLogicCqReport: RTS_DRIVER: report receive, stream_id=1, task_id=0, sq_id=1, sqHead=0, task_type=0(KERNEL_AICORE). +[ERROR] RUNTIME(3397738,execute_op):2024-08-19-10:21:36.550.575 [davinic_kernel_task.cc:1311]3397738 SetStarsResultForDavinciTask:AICORE Kernel task happen error, retCode=0x26. +``` + +## 解决思路 + +1.通过printf增加打印语句去定位报错代码块，该报错一般是访问了错误的地址，导致核函数执行异常。 +2.检查kernel侧代码是否存在逻辑问题，对buffer、Tensor访问是否超出本身申请的大小。 +3、构造相同输入，通过孪生调试功能的CPU侧调试进行定位，该方式可以直接显示错误原因，然后根据错误原因做具体分析，本例会输出类似： + +[TmSim]: Run in serial mode, HW concurrency is:192, actual thread num is: 1 + +[ERROR][Core_0][/usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/kernel_tpipe_impl.h:386]**[AllocBuffer][2971924] bufNum is 0, which must be larger than 0** +[ERROR][CORE_0][pid 2971924] error happened! ========= +SIGABRT Signal (Abort Signal from abort) catched, backtrace info: +[#0] 0x000000000041d2e4: Handler(int) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/lib/include/kern_fwk.h:106 +[#1] 0x00000000004128d4: AscendC::TQueBind<(AscendC::TPosition)0, (AscendC::TPosition)9, 2, 0>::AllocBuffer() at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/kernel_tpipe_impl.h:390 +[#2] 0x000000000041103c: AscendC::LocalTensor AscendC::TQueBind<(AscendC::TPosition)0, (AscendC::TPosition)9, 2, 0>::AllocTensor() at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/kernel_tpipe_impl.h:54 +[#3] 0x000000000040e65c: KernelAdd::CopyIn(int) at /root/libotao/ascend-error-example/MPUAddressAccessInvalid/AddKernelInvocation/**add_custom.cpp:46** +[#4] 0x000000000040e5fc: KernelAdd::Process() at /root/libotao/ascend-error-example/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp:37 (discriminator 2) +[#5] 0x000000000040b944: add_custom at /root/libotao/ascend-error-example/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp:80 +[#6] 0x000000000041cb20: main at /root/libotao/ascend-error-example/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp:34 (discriminator 127) +[#7] 0x000000000040b830: _start at ??:? + + + +根据加粗内容，AllocBuffers时超出TQue内最大Buffer数，TQue内只有0个Buffer，此时可能为未做InitBuffer操作，结合代码行数重点排查即可。 + diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/README.md b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/README.md new file mode 100755 index 000000000..69d8c877a --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/README.md @@ -0,0 +1,52 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ ├── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ ├── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ ├── operator_desc.cpp // 构造算子的输入与输出描述 +│ ├── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ``` + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ``` + cd $HOME/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ``` + bash run.sh + ``` +## 更新说明 + | 时间 | 更新事项 | +|----|------| +| 2024/5/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/common.h new file mode 100755 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/op_runner.h new file mode 100755 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/operator_desc.h new file mode 100755 index 000000000..4cbdf07e0 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/run.sh new file mode 100755 index 000000000..d7fd76334 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/run.sh @@ -0,0 +1,99 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + ./execute_add_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret=`python3 scripts/verify_result.py output/output_z.bin output/golden.bin` + echo $ret + if [ "x$ret" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/acl.json new file mode 100755 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/gen_data.py new file mode 100755 index 000000000..7babb0e76 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd +import numpy as np +import os +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/verify_result.py new file mode 100755 index 000000000..2528c9f07 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,23 @@ +import os +import sys +import numpy as np + +loss = 1e-3 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.float16) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.float16) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/CMakeLists.txt new file mode 100755 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp new file mode 100755 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/main.cpp new file mode 100755 index 000000000..a72e89598 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/main.cpp @@ -0,0 +1,163 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape { 8, 2048 }; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/op_runner.cpp new file mode 100755 index 000000000..cfad545ff --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,452 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_add_custom.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], + &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/operator_desc.cpp new file mode 100755 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakePresets.json new file mode 100755 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/build.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/config.cmake new file mode 100755 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..a14913c65 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b", "ascend910c"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c8b319944 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910c1": "ascend910c", + "ascend910c2": "ascend910c", + "ascend910c3": "ascend910c", + "ascend910c4": "ascend910c", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100755 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100755 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp new file mode 100755 index 000000000..2ee711900 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,72 @@ + +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" +#include "graph/utils/type_utils.h" +#include "tiling/platform/platform_ascendc.h" +namespace optiling { +const uint32_t BLOCK_DIM = 8; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + TilingData tiling; + uint64_t ubSize; + auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); + ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); + std::cout<GetInputTensor(0)->GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +static graphStatus InferDataType(gert::InferDataTypeContext* context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType);; + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h new file mode 100755 index 000000000..4bb6d940f --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) + TILING_DATA_FIELD_DEF(uint32_t, totalLength); + TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp new file mode 100755 index 000000000..03c0c5a00 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * + * Function : z = x + y + * This sample is a very basic sample that implements vector add on Ascend plaform. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue + +class KernelAdd +{ +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); + + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) + { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + __aicore__ inline void Test_Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[i * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[i * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + xLocal = inQueueX.DeQue(); + yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + + for (int32_t k = 0; k < 202; k++) + { + zLocal = outQueueZ.DeQue(); + outQueueZ.EnQue(zLocal); + } + + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + zLocal = outQueueZ.DeQue(); + + // DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + DataCopy(zGm[i * this->tileLength], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + + // DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + zGm.SetValue(4000*4000*8,zLocal.GetValue(0)); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + + if (TILING_KEY_IS(1)) { + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); + } +} + diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/NoTilingKey/README.md b/operator_contrib/CommonMistake/NoTilingKey/README.md new file mode 100644 index 000000000..f9a77d255 --- /dev/null +++ b/operator_contrib/CommonMistake/NoTilingKey/README.md @@ -0,0 +1,38 @@ +## 样例说明 + +该样例用于该样例用于复现找不到对应tilingkey的Kernel。 + +## 测试说明 + +编译部署AddCustom后，运行AclNNlnvocation调用，即可复现错误，界面会输出 + +```c++ +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Copy input[1] success +[INFO] Create stream success +[INFO] Execute aclnnAddCustomGetWorkspaceSize success, workspace size 0 +[ERROR] Execute Operator failed. error code is 361001 +[ERROR] Run op failed +[INFO] Reset Device success +[INFO] Destory resource success +ERROR: acl executable run failed! please check your project! +``` + +状态值：361001->API内存调用npu runtime的接口异常。 + +## 解决思路 + +1.首先要定位问题，是否在tiling侧设置多个tilingkey，与之相对的在host侧有没有对应的接受代码。例如： + +```c +tiling: + context->SetTilingKey(2); +host: + if (TILING_KEY_IS(2)) +``` + +2.当检测为tilingkey问题，开发者可以自行调整，在host侧增加对应代码或者在tiling侧减少设置tilingkey代码。 \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/README.md b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/README.md new file mode 100755 index 000000000..69d8c877a --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/README.md @@ -0,0 +1,52 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ ├── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ ├── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ ├── operator_desc.cpp // 构造算子的输入与输出描述 +│ ├── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ``` + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ``` + cd $HOME/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ``` + bash run.sh + ``` +## 更新说明 + | 时间 | 更新事项 | +|----|------| +| 2024/5/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/common.h new file mode 100755 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/op_runner.h new file mode 100755 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/operator_desc.h new file mode 100755 index 000000000..4cbdf07e0 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/run.sh new file mode 100755 index 000000000..d7fd76334 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/run.sh @@ -0,0 +1,99 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + ./execute_add_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret=`python3 scripts/verify_result.py output/output_z.bin output/golden.bin` + echo $ret + if [ "x$ret" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/acl.json new file mode 100755 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/gen_data.py new file mode 100755 index 000000000..7babb0e76 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd +import numpy as np +import os +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/verify_result.py new file mode 100755 index 000000000..2528c9f07 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,23 @@ +import os +import sys +import numpy as np + +loss = 1e-3 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.float16) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.float16) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/CMakeLists.txt new file mode 100755 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp new file mode 100755 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/main.cpp new file mode 100755 index 000000000..a72e89598 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/main.cpp @@ -0,0 +1,163 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape { 8, 2048 }; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/op_runner.cpp new file mode 100755 index 000000000..cfad545ff --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,452 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_add_custom.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], + &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/operator_desc.cpp new file mode 100755 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/OpNameError/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/OpNameError/AddCustom/CMakePresets.json new file mode 100755 index 000000000..8b7181000 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/build.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/build.sh new file mode 100755 index 000000000..27d1968d1 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/build.sh @@ -0,0 +1,80 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/config.cmake new file mode 100755 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..4179dfd25 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/func.cmake @@ -0,0 +1,192 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} && echo $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..63b15c900 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..647801c07 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is " + kernel_name) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}], param_type_dynamic = {},\\ + mc2_ctx = None, param_type_list = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def _get_snake_str(s, i): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + return "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + return "_" + return "" + return "" + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + snake_case += _get_snake_str(s, i) + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD.format(self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), False, self.input_type + self.output_type, self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..b138047af --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, task_ration, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': key, \ + 'multiKernelType':1, 'binPath': objfile}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE':3, 'MIX_VECTOR_CORE':4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + task_ration = contents.get('taskRation') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, task_ration, os.path.join(soc, op_dir, bin_file_name), \ + config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..336b4cdb2 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..492034e49 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..7e8dc2c55 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import re +import glob +import json +import argparse + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..48efca2c6 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} conflicts with\ + the built-in variable name. Use a complex name or prefix the operator name.') + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return + with open(file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..ac931e5d3 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights，group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..4c5b099b7 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100755 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc new file mode 100755 index 000000000..21064c663 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustomCC") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("AddCustomCC") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp new file mode 100755 index 000000000..4dc5bcbde --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp @@ -0,0 +1,65 @@ + +#include "add_custom_cc_tiling.h" +#include "register/op_def_registry.h" + + +namespace optiling { + const uint32_t BLOCK_DIM = 8; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + AddCustomCCTilingData tiling; + uint32_t totalLength = context->GetInputTensor(0)->GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class AddCustom1 : public OpDef { +public: + explicit AddCustom1(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom1); +} diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h new file mode 100755 index 000000000..8b9058ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -0,0 +1,11 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(AddCustomCCTilingData) + TILING_DATA_FIELD_DEF(uint32_t, totalLength); + TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom1, AddCustomCCTilingData) +} diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp new file mode 100755 index 000000000..4b4ca7611 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -0,0 +1,80 @@ + +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X*)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y*)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z*)z + this->blockLength * GetBlockIdx(), this->blockLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, this->tileLength); + //DataCopy(zLocal,xLocal,this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + //DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom1(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/OpNameError/README.md b/operator_contrib/CommonMistake/OpNameError/README.md new file mode 100644 index 000000000..fcfef223f --- /dev/null +++ b/operator_contrib/CommonMistake/OpNameError/README.md @@ -0,0 +1,40 @@ +## 样例说明 + +该样例用于复现算子AclNN调用时出现#include"aclnn_算子名称.h"头文件找不到，也就是算子名称错误。 + +## 测试说明 + +编译部署AddCustom后，运行AclNNlnvocation调用，即可复现错误，界面会输出 + +```c++ +/root/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp:16:10: fatal error: aclnn_add_custom.h: No such file or directory + 16 | #include "aclnn_add_custom.h" + | ^~~~~~~~~~~~~~~~~~~~ +compilation terminated. +make[2]: *** [CMakeFiles/execute_add_op.dir/build.make:90: CMakeFiles/execute_add_op.dir/op_runner.cpp.o] Error 1 +make[1]: *** [CMakeFiles/Makefile2:83: CMakeFiles/execute_add_op.dir/all] Error 2 +make: *** [Makefile:136: all] Error 2 +ERROR: make failed! +``` + +关键字：找不到"aclnn_add_custom.h"此类错误大概率为算子命名错误，还有一种可能是在部署算子时使用自定义安装路径。 + +如果之前部署过AddCustom算子 则会出现以下错误 + +```c++ +/usr/bin/ld: CMakeFiles/execute_add_op.dir/op_runner.cpp.o: in function `OpRunner::RunOp()': +op_runner.cpp:(.text+0x1368): undefined reference to `aclnnAddCustomGetWorkspaceSize' +/usr/bin/ld: op_runner.cpp:(.text+0x1438): undefined reference to `aclnnAddCustom' +collect2: error: ld returned 1 exit status +make[2]: *** [CMakeFiles/execute_add_op.dir/build.make:145: /root/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation/output/execute_add_op] Error 1 +make[1]: *** [CMakeFiles/Makefile2:83: CMakeFiles/execute_add_op.dir/all] Error 2 +make: *** [Makefile:136: all] Error 2 +ERROR: make failed! +``` + +关键字：undefined reference to `aclnnAddCustomGetWorkspaceSize'以及 undefined reference to `aclnnAddCustom'，通过这两项大概率可以确定为算子命名错误，还有一种可能部署算子时使用自定义安装路径。 + +## 解决思路 + +1.更改生成算子工程的json文件，把算子名称修改成正确的算子名称，再利用msopgen工具重新生成正确算子工程，再把自己的代码填补进去。 + diff --git a/operator_contrib/CommonMistake/README.md b/operator_contrib/CommonMistake/README.md new file mode 100644 index 000000000..4615bfe44 --- /dev/null +++ b/operator_contrib/CommonMistake/README.md @@ -0,0 +1,16 @@ + +| 目录名称 | 功能描述 | 运行环境 | +| ------------------------------------------------------------ | ---------------------------------------------------- | -- | +| [AclTensorFreeFail](./AclTensorFreeFail) | 输出内存使用异常，导致内存无法正常释放样例 | Atlas 200/500 A2推理产品 | +| [AddressMisalign](./AddressMisalign) | 地址未对齐样例 | Atlas 200/500 A2推理产品| +| [TimeOverrunBaseline](./TimeOverrunBaseline) | 算子耗时超过基线样例 | Atlas 200/500 A2推理产品 | +| [ResultError](./ResultError) | 算子运行结果错误样例 | Atlas 200/500 A2推理产品 | +| [SegmentationFault](./SegmentationFault) | 算子调用程序Segmentation样例 |Atlas 200/500 A2推理产品 | +| [SynchronizeFail](./SynchronizeFail) | 算子实现内存使用异常导致流同步超时样例 | Atlas 200/500 A2推理产品 | +| [InvalidMemory](./InvalidMemory) | GM或UB数据访问越界样例 | Atlas 200/500 A2推理产品 | +| [OpNameError](./OpNameError) | 算子命名错误，调用失败调用样例 | Atlas 200/500 A2推理产品 | +| [NoTilingKey](./NoTilingKey) | 找不到对应tilingkey的kernel样例 | Atlas 200/500 A2推理产品 | +| [TimeOut](./TimeOut) | 算子执行超过30s，脚本主动退出程序避免卡死样例 | Atlas 200/500 A2推理产品| +| [MPUAddressAccessInvalid](./MPUAddressAccessInvalid) | 访问无效内存地址样例 |Atlas 200/500 A2推理产品 | +| [TypeDeletion](./TypeDeletion) | 测试数据类型未注册样例 | Atlas 200/500 A2推理产品| + diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/README.md b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/README.md new file mode 100644 index 000000000..b8167d88f --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/README.md @@ -0,0 +1,54 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ └── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ```cpp + // 获取算子使用的workspace空间大小 + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + // 执行算子 + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + 以命令行方式下载样例代码，master分支为例。 + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ```bash + bash run.sh + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2024/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..11bb4aeca --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** + * @file common.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..66b6ea2c3 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,180 @@ +/** + * @file op_runner.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "acl/acl.h" +#include "aclnn/acl_meta.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..6d8ee0905 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** + * @file operator_desc.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/run.sh new file mode 100644 index 000000000..e1cec0584 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while :; do + case "$1" in + # float16, float, int32 + -v | --dtype) + DTYPE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash +export DDK_PATH=$_ASCEND_INSTALL_PATH +export NPU_HOST_LIB=$_ASCEND_INSTALL_PATH/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR + rm -rf build + mkdir -p build + cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + ./execute_add_op + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output_z.bin output/golden.bin + if [ $? -ne 0 ]; then + echo "ERROR: verify result failed!" + return 1 + fi +} + +main diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..6d15edfe0 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..1a21d809a --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 100: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..992759c95 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/common.cpp @@ -0,0 +1,80 @@ +/** + * @file common.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "common.h" + +#include +#include +#include + +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + size_t writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..481a31df8 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/main.cpp @@ -0,0 +1,162 @@ +/** + * @file main.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include +#include +#include + +#include +#include + +#include "acl/acl.h" +#include "common.h" +#include "op_runner.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..7e1623aed --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,453 @@ +/** + * @file op_runner.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "op_runner.h" + +#include +#include + +#include "acl/acl_op_compiler.h" +#include "aclnn_add_custom.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = + aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), nullptr, 0, + GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = + aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), nullptr, 0, + GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = + aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..da04cf6c9 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,51 @@ +/** + * @file operator_desc.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "operator_desc.h" + +#include "common.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/ResultError/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/ResultError/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/build.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/ResultError/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..3a6671209 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,73 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 8; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..323f3076f --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,22 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..2685319f6 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,93 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + this->blockLength = totalLength / AscendC::GetBlockNum(); + this->tileNum = tileNum; + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * AscendC::GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * AscendC::GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * AscendC::GetBlockIdx(), this->blockLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); + AscendC::DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + AscendC::DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = inQueueY.DeQue(); + AscendC::LocalTensor zLocal = outQueueZ.AllocTensor(); + AscendC::Sub(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + AscendC::LocalTensor zLocal = outQueueZ.DeQue(); + AscendC::DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX, inQueueY; + AscendC::TQue outQueueZ; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + AscendC::GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, + uint8_t *workspace, uint8_t *tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/ResultError/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/ResultError/README.md b/operator_contrib/CommonMistake/ResultError/README.md new file mode 100644 index 000000000..e23f26208 --- /dev/null +++ b/operator_contrib/CommonMistake/ResultError/README.md @@ -0,0 +1,38 @@ +## 样例说明 +该样例用于复现算子结果不正确 + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` + + [INFO] Set device[0] success + [INFO] Get RunMode[0] success + [INFO] Init resource success + [INFO] Set input success + [INFO] Copy input[0] success + [INFO] Copy input[1] success + [INFO] Copy input[2] success + [INFO] Create stream success + [INFO] Execute GetWorkspaceSize success, workspace size 0 + [INFO] Execute Operator success + [INFO] Synchronize stream success. + [INFO] Run op success + [INFO] Reset Device success + [INFO] Destory resource success + INFO：acl executable run success + [ERROR] result error (由于输出结果可能多个，可能会显示多个“XXX result error”) +``` + +## 解决思路 + +通过修改通过修改AclNNInvocation/scripts/verify_result.py中的比对精度，调用所部署的算子区分是否是精度不够还是数据错误 +精度问题： +1.通过printf打印关键输出值，对比精度是否符合要求。（可以使用DumpTensor进行Tensor的输出，支持型号见文档） +2.减少使用GetValue获取大量单个数值进行累加操作，尽可能使用API进行操作 +3.对于不能使用DumpTensor的情况可以将中间过程的数据copy到输出GM中，并不再覆盖输出GM中的数据，在AclNNInvocation/scripts/verify_result.py中新增打印“output”数据 + +数据错误： +1.通过printf对tiling结构体传参值和逻辑分支进行打印，判断tiling结构体数值正确并逻辑分支无异常 +2.通过printf打印关键输出值，是否可以与预期相符合。（可以使用DumpTensor进行Tensor的输出，支持型号见文档） +3.对于不能使用DumpTensor的情况可以将中间过程的数据copy到输出GM中，并不再覆盖输出GM中的数据，在AclNNInvocation/scripts/verify_result.py中新增打印“output”数据 \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/README.md b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/README.md new file mode 100644 index 000000000..75fa533f7 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/README.md @@ -0,0 +1,54 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ └── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ```cpp + // 获取算子使用的workspace空间大小 + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + // 执行算子 + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ```bash + bash run.sh + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2024/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..11bb4aeca --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** + * @file common.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..66b6ea2c3 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,180 @@ +/** + * @file op_runner.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "acl/acl.h" +#include "aclnn/acl_meta.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..6d8ee0905 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** + * @file operator_desc.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/run.sh new file mode 100644 index 000000000..25433eb68 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while :; do + case "$1" in + # float16, float, int32 + -v | --dtype) + DTYPE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash +export DDK_PATH=$_ASCEND_INSTALL_PATH +export NPU_HOST_LIB=$_ASCEND_INSTALL_PATH/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR + rm -rf build + mkdir -p build + cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_add_op + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output_z.bin output/golden.bin + if [ $? -ne 0 ]; then + echo "ERROR: verify result failed!" + return 1 + fi +} + +main diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..92b5a5bcf --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..ddb6e4a31 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 0: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..992759c95 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/common.cpp @@ -0,0 +1,80 @@ +/** + * @file common.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "common.h" + +#include +#include +#include + +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + size_t writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..481a31df8 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/main.cpp @@ -0,0 +1,162 @@ +/** + * @file main.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include +#include +#include + +#include +#include + +#include "acl/acl.h" +#include "common.h" +#include "op_runner.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..7e1623aed --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,453 @@ +/** + * @file op_runner.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "op_runner.h" + +#include +#include + +#include "acl/acl_op_compiler.h" +#include "aclnn_add_custom.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = + aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), nullptr, 0, + GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = + aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), nullptr, 0, + GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = + aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..da04cf6c9 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,51 @@ +/** + * @file operator_desc.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "operator_desc.h" + +#include "common.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/build.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..870b01d62 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,83 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 1; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + + int64_t shape[6]; + for (uint32_t i = 0; i < 2; i++) { + int64_t *ss = &shape[i * 3]; + const gert::StorageShape* storageShape = context -> GetInputShape(i); + for (int j = 0; j < storageShape->GetStorageShape().GetDimNum(); j++) { + ss[2 * i + j] = storageShape->GetStorageShape().GetDim(j); + } + } + tiling.set_shape(shape); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..5f7f213a3 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,23 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +TILING_DATA_FIELD_DEF_ARR(int64_t, 6, shape); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..584092996 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,96 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, + uint8_t *workspace, uint8_t *tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/SegmentationFault/README.md b/operator_contrib/CommonMistake/SegmentationFault/README.md new file mode 100644 index 000000000..f6fdd839e --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/README.md @@ -0,0 +1,32 @@ +## 样例说明 +该样例用于复现算子AclNN调用时出现Segmentation fault错误 + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` + +INFO: make success! +INFO: execute op! +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Create stream success +timeout: the monitored command dumped core +run.sh: line 45: 3179266 Segmentation fault timeout 30 ./execute_op +``` + + +查看plog日志，无报错，最后输出的日志一般和tiling相关，本例： + +[DEBUG] GE(3268987,execute_add_op):2024-08-16-11:20:43.691.960 [tilingdata_base.cc:72]3268987 InitData:TilingDef::InitData, op TilingData, data size 56. + + +## 解决思路 +1、阅读代码，确认host侧函数有对应的返回值，重点关注TilingFunc是否有返回值 +2、通过添加printf打印，直接在host打印确认代码错误位置，然后确认是否有语法问题，例如本示例，使用tiling传入数组时，未正确设置数组调用算子时出现 Segmentation fault +3、通过ascendebug工具调试tiling，确认tiling生成是否正常，调试代码内容参照tiling_debug.py ，调试结果会显示tiling是否能正常生成，例如： + + [ERROR]: generate tiling data exit with return code: -11, \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/run_test.sh b/operator_contrib/CommonMistake/SegmentationFault/run_test.sh new file mode 100755 index 000000000..e40ecfb5a --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/run_test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +LOG_PATH=/var/log/npu/slog/debug +if [ -d "$LOG_PATH" ]; then + echo "清理日志，路径：$LOG_PATH" +else + LOG_PATH=~/ascend/log + echo "清理日志，路径：$LOG_PATH" +fi +rm -rf $LOG_PATH* +cd AddCustom;./build.sh;./build_out/*.run +cd ../AclNNInvocation;bash run.sh +cd $LOG_PATH; grep -rn errorStr \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py b/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py new file mode 100644 index 000000000..ec9ee641f --- /dev/null +++ b/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py @@ -0,0 +1,15 @@ +import ascendebug +import numpy as np +input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) +input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) +golden = (input_x + input_y).astype(np.float16) + +debug_op = ascendebug.create_debug_op('AddCustom', 'VectorCore', 'Ascend910B1') \ + .tensor_input('x', input_x) \ + .tensor_input('y', input_y) \ + .tensor_output('z', golden) + +op_executor = ascendebug.create_op_executor(debug_op=debug_op, work_dir='./debug_workspace', install_path='/usr/local/Ascend') + +customize_path ="/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize" +tiling_info = op_executor.run_custom_tiling(customize_path) \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/README.md b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/README.md new file mode 100644 index 000000000..75fa533f7 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/README.md @@ -0,0 +1,54 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ └── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ```cpp + // 获取算子使用的workspace空间大小 + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + // 执行算子 + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ```bash + bash run.sh + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2024/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..11bb4aeca --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** + * @file common.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..66b6ea2c3 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,180 @@ +/** + * @file op_runner.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "acl/acl.h" +#include "aclnn/acl_meta.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..6d8ee0905 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** + * @file operator_desc.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/run.sh new file mode 100644 index 000000000..25433eb68 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while :; do + case "$1" in + # float16, float, int32 + -v | --dtype) + DTYPE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash +export DDK_PATH=$_ASCEND_INSTALL_PATH +export NPU_HOST_LIB=$_ASCEND_INSTALL_PATH/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR + rm -rf build + mkdir -p build + cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_add_op + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output_z.bin output/golden.bin + if [ $? -ne 0 ]; then + echo "ERROR: verify result failed!" + return 1 + fi +} + +main diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..92b5a5bcf --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..1a21d809a --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 100: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..992759c95 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/common.cpp @@ -0,0 +1,80 @@ +/** + * @file common.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "common.h" + +#include +#include +#include + +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + size_t writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..481a31df8 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/main.cpp @@ -0,0 +1,162 @@ +/** + * @file main.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include +#include +#include + +#include +#include + +#include "acl/acl.h" +#include "common.h" +#include "op_runner.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..7e1623aed --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,453 @@ +/** + * @file op_runner.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "op_runner.h" + +#include +#include + +#include "acl/acl_op_compiler.h" +#include "aclnn_add_custom.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = + aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), nullptr, 0, + GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = + aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), nullptr, 0, + GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = + aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..da04cf6c9 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,51 @@ +/** + * @file operator_desc.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "operator_desc.h" + +#include "common.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/build.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..6fc77c881 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,73 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 1; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..323f3076f --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,22 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..e3cd2837d --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,97 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + uint32_t loopCount =this->tileNum * BUFFER_NUM; + + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + // outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, + uint8_t *workspace, uint8_t *tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/CMakeLists.txt new file mode 100644 index 000000000..fd87c7620 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.16) +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_CXX_STANDARD 17) + +set(CCE_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules) +list(APPEND CMAKE_MODULE_PATH ${CCE_CMAKE_PATH}) +project(kernel_samples LANGUAGES CCE CXX) + +add_subdirectory(cmake/cpu) +add_subdirectory(cmake/npu) \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/README.md b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/README.md new file mode 100644 index 000000000..49bee5787 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/README.md @@ -0,0 +1,81 @@ +## 目录结构介绍 +``` +├── AddKernelInvocation +│ ├── cmake // 编译工程文件 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 验证输出数据和真值数据是否一致的验证脚本 +│ │ └── verify_result.py // 真值对比文件 +│ │── add_custom.cpp // 算子kernel实现 +│ │── CMakeLists.txt // 编译工程文件 +│ │── data_utils.h // 数据读入写出函数 +│ │── main.cpp // 主函数，调用算子的应用程序，含CPU域及NPU域调用 +│ └── run.sh // 编译运行算子的脚本 +``` +## 代码实现介绍 +本调用样例中实现的是固定shape为8*2048的Add算子。 +- kernel实现 + Add算子的数学表达式为： + ``` + z = x + y + ``` + 计算逻辑是：Ascend C提供的矢量计算接口的操作元素都为LocalTensor，输入数据需要先搬运进片上存储，然后使用计算接口完成两个输入参数相加，得到最终结果，再搬出到外部存储上。 + + Add算子的实现流程分为3个基本任务：CopyIn，Compute，CopyOut。CopyIn任务负责将Global Memory上的输入Tensor xGm和yGm搬运到Local Memory，分别存储在xLocal、yLocal，Compute任务负责对xLocal、yLocal执行加法操作，计算结果存储在zLocal中，CopyOut任务负责将输出数据从zLocal搬运至Global Memory上的输出Tensor zGm中。具体请参考[add_custom.cpp](./add_custom.cpp)。 + +- 调用实现 + 1. CPU侧运行验证主要通过ICPU_RUN_KF CPU调测宏等CPU调测库提供的接口来完成； + 2. NPU侧运行验证主要通过使用<<<>>>内核调用符来完成。 + + 应用程序通过ASCENDC_CPU_DEBUG 宏区分代码逻辑运行于CPU侧还是NPU侧。 + +## 运行样例算子 + - 打开样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/KernelLaunch/AddKernelInvocation + ``` + - 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ```` + + 配置仿真模式日志文件目录，默认为sim_log。 + ```bash + export CAMODEL_LOG_PATH=./sim_log + ``` + + - 样例执行 + + ```bash + bash run.sh -r [RUN_MODE] -v [SOC_VERSION] + ``` + - RUN_MODE：编译方式，可选择CPU调试，NPU仿真，NPU上板。支持参数为[cpu / sim / npu]，默认值为cpu。 + - SOC_VERSION：昇腾AI处理器型号，如果无法确定具体的[SOC_VERSION]，则在安装昇腾AI处理器的服务器执行npu-smi info命令进行查询，在查询到的“Name”前增加Ascend信息，例如“Name”对应取值为xxxyy，实际配置的[SOC_VERSION]值为Ascendxxxyy。支持以下参数取值（xxx请替换为具体取值）： + - Atlas 推理系列产品（Ascend 310P处理器）参数值：Ascend310P1、Ascend310P3 + - Atlas 训练系列产品参数值：AscendxxxA、AscendxxxB + - Atlas A2训练系列产品参数值：AscendxxxB1、AscendxxxB2、AscendxxxB3、AscendxxxB4 + + 注：针对Atlas 训练系列产品使用NPU仿真调试，会存在精度问题，可选择其他芯片进行NPU仿真调试。 + + 示例如下。 + ```bash + bash run.sh -r cpu -v Ascend310P1 + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2023/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp new file mode 100644 index 000000000..9ee4fb6df --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp @@ -0,0 +1,91 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; + +constexpr int32_t TOTAL_LENGTH = 8 * 2048; // total length of data +constexpr int32_t USE_CORE_NUM = 1; // num of core used +constexpr int32_t BLOCK_LENGTH = TOTAL_LENGTH / USE_CORE_NUM; // length computed of each core +constexpr int32_t TILE_NUM = 8; // split data into 8 tiles for each core +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue +constexpr int32_t TILE_LENGTH = BLOCK_LENGTH / TILE_NUM / BUFFER_NUM; // seperate to 2 parts, due to double buffer + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z) + { + xGm.SetGlobalBuffer((__gm__ half *)x + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + yGm.SetGlobalBuffer((__gm__ half *)y + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + zGm.SetGlobalBuffer((__gm__ half *)z + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); + pipe.InitBuffer(inQueueX, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, TILE_LENGTH * sizeof(half)); + } + __aicore__ inline void Process() + { + int32_t loopCount = TILE_NUM * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * TILE_LENGTH], TILE_LENGTH); + DataCopy(yLocal, yGm[progress * TILE_LENGTH], TILE_LENGTH); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, TILE_LENGTH); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * TILE_LENGTH], zLocal, TILE_LENGTH); + // outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +{ + KernelAdd op; + op.Init(x, y, z); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +{ + add_custom<<>>(x, y, z); +} +#endif diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in new file mode 100644 index 000000000..a9b5688ff --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCECompiler.cmake.in @@ -0,0 +1,5 @@ +set(CMAKE_CCE_COMPILER "@CMAKE_CCE_COMPILER@") +set(CMAKE_CCE_COMPILER_LOADED 1) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS @CMAKE_CCE_SOURCE_FILE_EXTENSIONS@) +set(CMAKE_CCE_OUTPUT_EXTENSION @CMAKE_CCE_OUTPUT_EXTENSION@) +set(CMAKE_CCE_COMPILER_ENV_VAR "@CMAKE_CCE_COMPILER_ENV_VAR@") diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake new file mode 100644 index 000000000..7d2fc2ddd --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeCCEInformation.cmake @@ -0,0 +1,41 @@ +include(CMakeCommonLanguageInclude) + +set(CMAKE_INCLUDE_FLAG_CCE "-I") + +if(UNIX) + set(CMAKE_CCE_OUTPUT_EXTENSION .o) +else() + set(CMAKE_CCE_OUTPUT_EXTENSION .obj) +endif() + +set(_INCLUDED_FILE 0) +set(CMAKE_SHARED_LIBRARY_CCE_FLAGS -fPIC) +set(CMAKE_SHARED_LIBRARY_CREATE_CCE_FLAGS -shared) +set(CMAKE_STATIC_LIBRARY_CREATE_CCE_FLAGS "--cce-build-static-lib") +set(CMAKE_LIBRARY_CREATE_CCE_FLAGS "--cce-fatobj-link") + +if(NOT CMAKE_CCE_COMPILE_OBJECT) + set(CMAKE_CCE_COMPILE_OBJECT + " -xcce ${__IMPLICIT_INCLUDES} ${_CMAKE_CCE_BUILTIN_INCLUDE_PATH} ${_CMAKE_COMPILE_AS_CCE_FLAG} ${_CMAKE_CCE_COMPILE_OPTIONS} ${_CMAKE_CCE_COMMON_COMPILE_OPTIONS} -pthread -o -c ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_LIBRARY) + set(CMAKE_CCE_CREATE_SHARED_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_STATIC_LIBRARY) + set(CMAKE_CCE_CREATE_STATIC_LIBRARY + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ") +endif() + +if(NOT CMAKE_CCE_CREATE_SHARED_MODULE) + set(CMAKE_CCE_CREATE_SHARED_MODULE ${CMAKE_CCE_CREATE_SHARED_LIBRARY}) +endif() + +if(NOT CMAKE_CCE_LINK_EXECUTABLE) + set(CMAKE_CCE_LINK_EXECUTABLE + " ${CMAKE_LIBRARY_CREATE_CCE_FLAGS} -o ${__IMPLICIT_LINKS}") +endif() + +set(CMAKE_CCE_INFORMATION_LOADED 1) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake new file mode 100755 index 000000000..f3c0a4732 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeDetermineCCECompiler.cmake @@ -0,0 +1,124 @@ +find_program(CMAKE_CCE_COMPILER NAMES "ccec" PATHS "$ENV{PATH}" DOC "CCE Compiler") + +mark_as_advanced(CMAKE_CCE_COMPILER) + +message(STATUS "CMAKE_CCE_COMPILER: " ${CMAKE_CCE_COMPILER}) +set(CMAKE_CCE_SOURCE_FILE_EXTENSIONS cce;cpp) +set(CMAKE_CCE_COMPILER_ENV_VAR "CCE") +message(STATUS "CMAKE_CURRENT_LIST_DIR: " ${CMAKE_CURRENT_LIST_DIR}) + +# configure all variables set in this file +configure_file(${CMAKE_CURRENT_LIST_DIR}/CMakeCCECompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeCCECompiler.cmake + @ONLY +) + +message(STATUS "ASCEND_PRODUCT_TYPE:\n" " ${ASCEND_PRODUCT_TYPE}") +message(STATUS "ASCEND_CORE_TYPE:\n" " ${ASCEND_CORE_TYPE}") +message(STATUS "ASCEND_INSTALL_PATH:\n" " ${ASCEND_INSTALL_PATH}") + +if(DEFINED ASCEND_INSTALL_PATH) + set(_CMAKE_ASCEND_INSTALL_PATH ${ASCEND_INSTALL_PATH}) +else() + message(FATAL_ERROR + "no, installation path found, should passing -DASCEND_INSTALL_PATH= in cmake" + ) + set(_CMAKE_ASCEND_INSTALL_PATH) +endif() + + +if(DEFINED ASCEND_PRODUCT_TYPE) + set(_CMAKE_CCE_COMMON_COMPILE_OPTIONS "--cce-auto-sync -mllvm -api-deps-filter") + if(ASCEND_PRODUCT_TYPE STREQUAL "") + message(FATAL_ERROR "ASCEND_PRODUCT_TYPE must be non-empty if set.") + elseif(ASCEND_PRODUCT_TYPE AND NOT ASCEND_PRODUCT_TYPE MATCHES "^Ascend[0-9][0-9][0-9][a-zA-Z]?[1-9]?$") + message(FATAL_ERROR + "ASCEND_PRODUCT_TYPE: ${ASCEND_PRODUCT_TYPE}\n" + "is not one of the following: Ascend910A, Ascend310P1, Ascend910B1, Ascend310B1" + ) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c100") + else() + message(FATAL_ERROR, "only AiCore inside") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS) + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310P1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m200-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend310B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-m300-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-fp-ceiling=2 -mllvm -cce-aicore-record-overflow=false") + elseif(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910B1") + if (ASCEND_CORE_TYPE STREQUAL "AiCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-cube") + elseif(ASCEND_CORE_TYPE STREQUAL "VectorCore") + set(_CMAKE_COMPILE_AS_CCE_FLAG "--cce-aicore-arch=dav-c220-vec") + endif() + set(_CMAKE_CCE_COMPILE_OPTIONS + "-mllvm -cce-aicore-function-stack-size=16000 -mllvm -cce-aicore-record-overflow=false -mllvm -cce-aicore-addr-transform") + endif() +endif() + +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/runtime/lib64 + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/simulator/${ASCEND_PRODUCT_TYPE}/lib + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/${ASCEND_PRODUCT_TYPE} +) + +# link library +set(_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES stdc++) +if(ASCEND_RUN_MODE STREQUAL "npu") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime) +elseif(ASCEND_RUN_MODE STREQUAL "sim") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES ) + if(ASCEND_PRODUCT_TYPE STREQUAL "Ascend910A") + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES pem_davinci) + endif() + list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES runtime_camodel) +elseif(ASCEND_RUN_MODE STREQUAL "cpu") + message(STATUS "RUN_MODE is cpu") +else() + message(FATAL_ERROR + "ASCEND_RUN_MODE: ${ASCEND_RUN_MODE}\n" + "ASCEND_RUN_MODE must be one of the following: cpu, npu or sim" + ) +endif() +list(APPEND _CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES ascendcl) + +set(__IMPLICIT_LINKS) +foreach(dir ${_CMAKE_CCE_HOST_IMPLICIT_LINK_DIRECTORIES}) + string(APPEND __IMPLICIT_LINKS " -L\"${dir}\"") +endforeach() +foreach(lib ${_CMAKE_CCE_HOST_IMPLICIT_LINK_LIBRARIES}) + if(${lib} MATCHES "/") + string(APPEND __IMPLICIT_LINKS " \"${lib}\"") + else() + string(APPEND __IMPLICIT_LINKS " -l${lib}") + endif() +endforeach() + +set(_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES + ${_CMAKE_ASCEND_INSTALL_PATH}/acllib/include + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/compiler/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/impl + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikcpp/tikcfw/interface + ${_CMAKE_ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/include +) +set(__IMPLICIT_INCLUDES) +foreach(inc ${_CMAKE_CCE_HOST_IMPLICIT_INCLUDE_DIRECTORIES}) + string(APPEND __IMPLICIT_INCLUDES " -I\"${inc}\"") +endforeach() \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake new file mode 100644 index 000000000..f00f227c1 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/Modules/CMakeTestCCECompiler.cmake @@ -0,0 +1 @@ +set(CMAKE_CCE_COMPILER_WORKS 1 CACHE INTERNAL "") diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt new file mode 100644 index 000000000..661230b17 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/cpu/CMakeLists.txt @@ -0,0 +1,37 @@ +# cpu +if (NOT DEFINED ENV{CMAKE_PREFIX_PATH}) + set(CMAKE_PREFIX_PATH ${ASCEND_INSTALL_PATH}/tools/tikicpulib/lib/cmake) +endif() + +find_package(tikicpulib REQUIRED) + +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) + +add_executable(${smoke_testcase}_cpu + ${SRC_FILES} +) + +target_include_directories(${smoke_testcase}_cpu PRIVATE + ${ASCEND_INSTALL_PATH}/acllib/include + ${CMAKE_SOURCE_DIR} +) + +target_link_libraries(${smoke_testcase}_cpu PRIVATE + tikicpulib::${ASCEND_PRODUCT_TYPE} + ascendcl +) + +target_compile_options(${smoke_testcase}_cpu PRIVATE + -g +) + +target_compile_definitions(${smoke_testcase}_cpu PRIVATE + _GLIBCXX_USE_CXX11_ABI=0 +) + +set_target_properties(${smoke_testcase}_cpu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt new file mode 100644 index 000000000..fb1786c6b --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/cmake/npu/CMakeLists.txt @@ -0,0 +1,26 @@ +# npu +file(GLOB SRC_FILES + ${CMAKE_SOURCE_DIR}/*.cpp +) +set_source_files_properties(${SRC_FILES} PROPERTIES LANGUAGE CCE) + +add_executable(${smoke_testcase}_npu + ${SRC_FILES} +) + +target_compile_options(${smoke_testcase}_npu PRIVATE + -O2 + -std=c++17 +) + +target_link_directories(${smoke_testcase}_npu PRIVATE + ${ASCEND_INSTALL_PATH}/lib64 +) + +set_target_properties(${smoke_testcase}_npu PROPERTIES + OUTPUT_NAME ${smoke_testcase}_${ASCEND_RUN_MODE} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR} +) + +add_custom_target(${smoke_testcase}_sim ALL) +add_dependencies(${smoke_testcase}_sim ${smoke_testcase}_npu) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/data_utils.h b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/data_utils.h new file mode 100644 index 000000000..7cdefd866 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/data_utils.h @@ -0,0 +1,203 @@ +/** + * @file data_utils.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef DATA_UTILS_H +#define DATA_UTILS_H +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +typedef enum { + DT_UNDEFINED = -1, + FLOAT = 0, + HALF = 1, + INT8_T = 2, + INT32_T = 3, + UINT8_T = 4, + INT16_T = 6, + UINT16_T = 7, + UINT32_T = 8, + INT64_T = 9, + UINT64_T = 10, + DOUBLE = 11, + BOOL = 12, + STRING = 13, + COMPLEX64 = 16, + COMPLEX128 = 17, + BF16 = 27 +} printDataType; + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stdout, "[ERROR] " fmt "\n", ##args) +#define CHECK_ACL(x) \ + do { \ + aclError __ret = x; \ + if (__ret != ACL_ERROR_NONE) { \ + std::cerr << __FILE__ << ":" << __LINE__ << " aclError:" << __ret << std::endl; \ + } \ + } while (0); + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t &fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file"); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintHalfData(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(6) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, printDataType dataType, size_t elementsPerRow = 16) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT8_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT16_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT32_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case INT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case UINT64_T: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case HALF: + DoPrintHalfData(reinterpret_cast(data), count, elementsPerRow); + break; + case FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } + std::cout << std::endl; +} +#endif // DATA_UTILS_H diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp new file mode 100644 index 000000000..bd760f1c6 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp @@ -0,0 +1,82 @@ +/** + * @file main.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "data_utils.h" +#ifndef ASCENDC_CPU_DEBUG +#include "acl/acl.h" +extern void add_custom_do(uint32_t coreDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z); +#else +#include "tikicpulib.h" +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z); +#endif + +int32_t main(int32_t argc, char *argv[]) +{ + uint32_t blockDim = 1; + size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + +#ifdef ASCENDC_CPU_DEBUG + uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *y = (uint8_t *)AscendC::GmAlloc(inputByteSize); + uint8_t *z = (uint8_t *)AscendC::GmAlloc(outputByteSize); + + ReadFile("./input/input_x.bin", inputByteSize, x, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, y, inputByteSize); + + AscendC::SetKernelMode(KernelMode::AIV_MODE); + ICPU_RUN_KF(add_custom, blockDim, x, y, z); // use this macro for cpu debug + + WriteFile("./output/output_z.bin", z, outputByteSize); + + AscendC::GmFree((void *)x); + AscendC::GmFree((void *)y); + AscendC::GmFree((void *)z); +#else + CHECK_ACL(aclInit("./scripts/acl.json")); + int32_t deviceId = 0; + CHECK_ACL(aclrtSetDevice(deviceId)); + aclrtStream stream = nullptr; + CHECK_ACL(aclrtCreateStream(&stream)); + + uint8_t *xHost, *yHost, *zHost; + uint8_t *xDevice, *yDevice, *zDevice; + + CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); + CHECK_ACL(aclrtMallocHost((void **)(&zHost), outputByteSize)); + CHECK_ACL(aclrtMalloc((void **)&xDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&yDevice, inputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + CHECK_ACL(aclrtMalloc((void **)&zDevice, outputByteSize, ACL_MEM_MALLOC_HUGE_FIRST)); + + ReadFile("./input/input_x.bin", inputByteSize, xHost, inputByteSize); + ReadFile("./input/input_y.bin", inputByteSize, yHost, inputByteSize); + + CHECK_ACL(aclrtMemcpy(xDevice, inputByteSize, xHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + CHECK_ACL(aclrtMemcpy(yDevice, inputByteSize, yHost, inputByteSize, ACL_MEMCPY_HOST_TO_DEVICE)); + + add_custom_do(blockDim, nullptr, stream, xDevice, yDevice, zDevice); + CHECK_ACL(aclrtSynchronizeStream(stream)); + + CHECK_ACL(aclrtMemcpy(zHost, outputByteSize, zDevice, outputByteSize, ACL_MEMCPY_DEVICE_TO_HOST)); + WriteFile("./output/output_z.bin", zHost, outputByteSize); + + CHECK_ACL(aclrtFree(xDevice)); + CHECK_ACL(aclrtFree(yDevice)); + CHECK_ACL(aclrtFree(zDevice)); + CHECK_ACL(aclrtFreeHost(xHost)); + CHECK_ACL(aclrtFreeHost(yHost)); + CHECK_ACL(aclrtFreeHost(zHost)); + + CHECK_ACL(aclrtDestroyStream(stream)); + CHECK_ACL(aclrtResetDevice(deviceId)); + CHECK_ACL(aclFinalize()); +#endif + return 0; +} diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/run.sh b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/run.sh new file mode 100644 index 000000000..e741e1224 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/run.sh @@ -0,0 +1,130 @@ +#!/bin/bash +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +declare -A VersionMap +VersionMap["Ascend910A"]="Ascend910A" +VersionMap["Ascend910B"]="Ascend910A" +VersionMap["Ascend910ProA"]="Ascend910A" +VersionMap["Ascend910ProB"]="Ascend910A" +VersionMap["Ascend910PremiumA"]="Ascend910A" +VersionMap["Ascend310B1"]="Ascend310B1" +VersionMap["Ascend310B2"]="Ascend310B1" +VersionMap["Ascend310B3"]="Ascend310B1" +VersionMap["Ascend310B4"]="Ascend310B1" +VersionMap["Ascend310P1"]="Ascend310P1" +VersionMap["Ascend310P3"]="Ascend310P1" +VersionMap["Ascend910B1"]="Ascend910B1" +VersionMap["Ascend910B2"]="Ascend910B1" +VersionMap["Ascend910B3"]="Ascend910B1" +VersionMap["Ascend910B4"]="Ascend910B1" +# legacy +VersionMap["ascend910"]="Ascend910A" +VersionMap["ascend310p"]="Ascend310P1" +VersionMap["ascend310B1"]="Ascend310B1" +VersionMap["ascend910B1"]="Ascend910B1" + +FILE_NAME="add" + +SHORT=r:,v:,i:, +LONG=run-mode:,soc-version:,install-path:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" + +while :; do + case "$1" in + -r | --run-mode) + RUN_MODE="$2" + shift 2 + ;; + -v | --soc-version) + SOC_VERSION="$2" + shift 2 + ;; + -i | --install-path) + ASCEND_INSTALL_PATH="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +# in case of running op in simulator, use stub so instead +if [ "${RUN_MODE}" = "sim" ]; then + export LD_LIBRARY_PATH=$_ASCEND_INSTALL_PATH/runtime/lib64/stub:$LD_LIBRARY_PATH + if [ ! $CAMODEL_LOG_PATH ]; then + export CAMODEL_LOG_PATH=./sim_log + fi + rm -rf $CAMODEL_LOG_PATH + mkdir -p $CAMODEL_LOG_PATH +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash + +if [[ " ${!VersionMap[*]} " != *" $SOC_VERSION "* ]]; then + echo "ERROR: SOC_VERSION should be in [${!VersionMap[*]}]" + exit -1 +fi +_SOC_VERSION=${VersionMap[$SOC_VERSION]} + +if [ $_SOC_VERSION"x" = "Ascend910Ax" ] || [ $_SOC_VERSION"x" = "Ascend310P1x" ] || [ $_SOC_VERSION"x" = "Ascend310B1x" ]; then + CORE_TYPE="AiCore" +elif [ $_SOC_VERSION"x" = "Ascend910B1x" ]; then + CORE_TYPE="VectorCore" +fi + +RUN_MODE_LIST="cpu sim npu" +if [[ " $RUN_MODE_LIST " != *" $RUN_MODE "* ]]; then + echo "ERROR: RUN_MODE error, This sample only support specify cpu, sim or npu!" + exit -1 +fi + +set -e +rm -rf build *_cpu *_sim *_npu cceprint npuchk *log *.vcd + +rm -rf build +mkdir -p build +cmake -B build \ + -Dsmoke_testcase=${FILE_NAME} \ + -DASCEND_PRODUCT_TYPE=${_SOC_VERSION} \ + -DASCEND_CORE_TYPE=${CORE_TYPE} \ + -DASCEND_RUN_MODE=${RUN_MODE} \ + -DASCEND_INSTALL_PATH=${_ASCEND_INSTALL_PATH} +cmake --build build --target ${FILE_NAME}_${RUN_MODE} +if [ $? -ne 0 ]; then + echo "ERROR: compile op on failed!" + exit -1 +fi +echo "INFO: compile op on ${RUN_MODE} succeed!" + +rm -rf input/*.bin output/*.bin +python3 scripts/gen_data.py +(export LD_LIBRARY_PATH=${_ASCEND_INSTALL_PATH}/tools/simulator/${_SOC_VERSION}/lib:$LD_LIBRARY_PATH && ./${FILE_NAME}_${RUN_MODE}) +if [ $? -ne 0 ]; then + echo "ERROR: execute op on ${RUN_MODE} failed!" + exit -1 +fi +echo "INFO: execute op on ${RUN_MODE} succeed!" +python3 scripts/verify_result.py output/output_z.bin output/golden.bin + +rm -rf *log *.vcd diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/acl.json b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/gen_data.py new file mode 100644 index 000000000..a85a84a15 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/gen_data.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/verify_result.py new file mode 100644 index 000000000..ddb6e4a31 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 0: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/SynchronizeFail/README.md b/operator_contrib/CommonMistake/SynchronizeFail/README.md new file mode 100644 index 000000000..18512ace6 --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/README.md @@ -0,0 +1,56 @@ +## 样例说明 +该样例用于AclNN调用算子流同步失败错误 + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` + +INFO: make success! +INFO: execute op! +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Copy input[1] success +[INFO] Create stream success +[INFO] Execute GetWorkspaceSize success, workspace size 0 +[INFO] Execute Operator success +[ERROR] Synchronize stream failed. error code is 507046 +[ERROR] Run op failed +[INFO] Reset Device success +[INFO] Destory resource success + + +``` + +使用grep在plog日志查找ERROR关键字有类似信息 + +[ERROR] RUNTIME(3267549,execute_add_op):2024-08-16-11:11:41.780.751 [api_c_stream.cc:145]3267549 rtStreamSynchronizeWithTimeout:ErrCode=507046, desc=[stream sync timeout], InnerCode=0x7030010 + + +## 解决思路 +1、阅读代码，检查是否有TQue申请的内存未释放 +2、通过添加printf打印，确认代码执行阻塞位置，再具体分析（需要产品支持printf调试API） +3、构造相同输入，通过孪生调试功能的CPU侧调试进行定位，该方式可以直接显示错误原因，再做具体分析，本例会输出类似： + +[TmSim]: Run in serial mode. +[ERROR][Block_0][AIV_0][/usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/kernel_tpipe_impl.h:440]**[AllocBuffer][3362940] size is 3, which exceed limits 2** +[ERROR][CORE_0][pid 3362940] error happened! ========= +SIGABRT Signal (Abort Signal from abort) catched, backtrace info: +[#0] 0x000000000041c120: handler(int) at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/lib/include/kern_fwk.h:98 +[#1] 0x0000000000413134: AscendC::TQueBind<(AscendC::TPosition)10, (AscendC::TPosition)0, 2, 0>::AllocBuffer() at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/kernel_tpipe_impl.h:441 (discriminator 6) +[#2] 0x0000000000410b6c: AscendC::LocalTensor AscendC::TQueBind<(AscendC::TPosition)10, (AscendC::TPosition)0, 2, 0>::AllocTensor() at /usr/local/Ascend/ascend-toolkit/latest/tools/tikicpulib/../../compiler/tikcpp/tikcfw/impl/kernel_tpipe_impl.h:64 +[#3] 0x000000000040e254:** KernelAdd::Compute(int) at /root/libotao/ascend-error-example/SynchronizeFail/AddKernelInvocation/add_custom.cpp:57** +[#4] 0x000000000040e0c0: KernelAdd::Process() at /root/libotao/ascend-error-example/SynchronizeFail/AddKernelInvocation/add_custom.cpp:38 (discriminator 2) +[#5] 0x000000000040ac84: add_custom at /root/libotao/ascend-error-example/SynchronizeFail/AddKernelInvocation/add_custom.cpp:80 +[#6] 0x000000000041bcc8: main at /root/libotao/ascend-error-example/SynchronizeFail/AddKernelInvocation/main.cpp:34 (discriminator 127) +[#7] 0x000000000040ab70: _start at ??:? + + +如上输出可以根据加粗内容分析： + 第一个加粗内容为分配内存失败，原因是超出了可分配内存块数限制，可分配内存块数根据创建TQue对象时传的depth决定，本样例对应代码为TQue outQueueZ; 那么最多可以分配BUFFER_NUM块，如果超出则可能为只调用AllocTensor，没有匹配成对的FreeTensor。 + 第二个加粗内容为第57行使用了该分配失败的内存LocalTensor对象，可以看到代码57行为Add(zLocal, xLocal, yLocal, TILE_LENGTH);那么可以针对性分析zLocal, xLocal, yLocal确认是哪个对应的TQue未做释放操作。 + + diff --git a/operator_contrib/CommonMistake/SynchronizeFail/run_test.sh b/operator_contrib/CommonMistake/SynchronizeFail/run_test.sh new file mode 100755 index 000000000..5facccd0a --- /dev/null +++ b/operator_contrib/CommonMistake/SynchronizeFail/run_test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +LOG_PATH=/var/log/npu/slog/debug +if [ -d "$LOG_PATH" ]; then + echo "清理日志，路径：$LOG_PATH" +else + LOG_PATH=~/ascend/log + echo "清理日志，路径：$LOG_PATH" +fi +rm -rf $LOG_PATH* +cd AddCustom;./build.sh;./build_out/*.run +cd ../AclNNInvocation;bash run.sh +cd $LOG_PATH; grep -rn ERROR \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/README.md b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/README.md new file mode 100755 index 000000000..69d8c877a --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/README.md @@ -0,0 +1,52 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ ├── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ ├── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ ├── operator_desc.cpp // 构造算子的输入与输出描述 +│ ├── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ``` + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ``` + cd $HOME/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ``` + bash run.sh + ``` +## 更新说明 + | 时间 | 更新事项 | +|----|------| +| 2024/5/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/common.h new file mode 100755 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/op_runner.h new file mode 100755 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/operator_desc.h new file mode 100755 index 000000000..4cbdf07e0 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/run.sh new file mode 100755 index 000000000..cd0fc9af0 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/run.sh @@ -0,0 +1,116 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + # rm ./input/*.bin + rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_add_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret=`python3 scripts/verify_result.py output/output_z.bin output/golden.bin` + echo $ret + if [ "x$ret" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/acl.json new file mode 100755 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/gen_data.py new file mode 100755 index 000000000..7babb0e76 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd +import numpy as np +import os +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/verify_result.py new file mode 100755 index 000000000..2528c9f07 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,23 @@ +import os +import sys +import numpy as np + +loss = 1e-3 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.float16) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.float16) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/CMakeLists.txt new file mode 100755 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp new file mode 100755 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/main.cpp new file mode 100755 index 000000000..a72e89598 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/main.cpp @@ -0,0 +1,163 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape { 8, 2048 }; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/op_runner.cpp new file mode 100755 index 000000000..cfad545ff --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,452 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_add_custom.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], + &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/operator_desc.cpp new file mode 100755 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOut/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/TimeOut/AddCustom/CMakePresets.json new file mode 100755 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/build.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/config.cmake new file mode 100755 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..a14913c65 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b", "ascend910c"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c8b319944 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910c1": "ascend910c", + "ascend910c2": "ascend910c", + "ascend910c3": "ascend910c", + "ascend910c4": "ascend910c", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100755 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100755 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp new file mode 100755 index 000000000..7c33bce2e --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,71 @@ + +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" +#include "graph/utils/type_utils.h" +#include "tiling/platform/platform_ascendc.h" +namespace optiling { +const uint32_t BLOCK_DIM = 8; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + TilingData tiling; + uint64_t ubSize; + auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); + ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); + uint32_t totalLength = context->GetInputTensor(0)->GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + + while(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +static graphStatus InferDataType(gert::InferDataTypeContext* context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType);; + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h new file mode 100755 index 000000000..4bb6d940f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) + TILING_DATA_FIELD_DEF(uint32_t, totalLength); + TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp new file mode 100755 index 000000000..405cc38dd --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * + * Function : z = x + y + * This sample is a very basic sample that implements vector add on Ascend plaform. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue + +class KernelAdd +{ +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); + + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) + { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} +/* +#ifndef __CCE_KT_TEST__ +// call of kernel function +void add_custom_do(uint32_t blockDim, void* l2ctrl, void* stream, uint8_t* x, uint8_t* y, uint8_t* z, + uint8_t* workspace, uint8_t* tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif*/ diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/TimeOut/README.md b/operator_contrib/CommonMistake/TimeOut/README.md new file mode 100644 index 000000000..ac1b2d5b4 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOut/README.md @@ -0,0 +1,23 @@ +## 样例说明 + +该样例用于复现算子AclNN调用时出现卡在创建流，不出现结果。 + +## 测试说明 + +编译部署AddCustom后，运行AclNNlnvocation调用，即可复现错误。 + +```c +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Copy input[1] success +[INFO] Create stream success +``` + + + +## 解决思路 + +1.在host侧可能存在死循环，需要在host侧增加打印，看是哪一行代码出了问题。 diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/README.md b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/README.md new file mode 100644 index 000000000..b8167d88f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/README.md @@ -0,0 +1,54 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ └── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ```cpp + // 获取算子使用的workspace空间大小 + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + // 执行算子 + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + 以命令行方式下载样例代码，master分支为例。 + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ```bash + bash run.sh + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2024/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..11bb4aeca --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** + * @file common.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..66b6ea2c3 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,180 @@ +/** + * @file op_runner.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "acl/acl.h" +#include "aclnn/acl_meta.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..6d8ee0905 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** + * @file operator_desc.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/run.sh new file mode 100644 index 000000000..e376edae7 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/run.sh @@ -0,0 +1,119 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while :; do + case "$1" in + # float16, float, int32 + -v | --dtype) + DTYPE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash +export DDK_PATH=$_ASCEND_INSTALL_PATH +export NPU_HOST_LIB=$_ASCEND_INSTALL_PATH/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR + rm -rf build + mkdir -p build + cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 msprof op --launch-skip-before-match=0 ./execute_add_op + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + time_ust=$(($(python3 $CURRENT_DIR/scripts/get_time.py))) + time_base=3720000 + echo "time_base = $time_base time_use = $time_ust" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret=`python3 scripts/verify_result.py output/output_z.bin output/golden.bin` + echo $ret + if [ "x$ret" == "xtest pass" ]; then + + if [ $time_ust -eq 0 ]; then + echo "[ERROR] Performance not achieved" + return 1 + fi + + if [ $time_ust -ge $time_base ]; then + echo "test fail for performance exceeds baseline data" + return 1 + fi + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..6d15edfe0 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) + golden = (input_x + input_y).astype(np.float16) + os.system("mkdir -p input output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py new file mode 100644 index 000000000..fbeac398d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py @@ -0,0 +1,31 @@ +import csv +from pathlib import Path + + +def get_time(file_path, time_use_list): + with open(file_path, 'r', encoding='utf-8') as file: + reader = csv.DictReader(file) + for row in reader: + time_use = row['Task Duration(us)'] + time_use_list.append(int(float(time_use)* 1000000)) + + +def find_min_time(): + min_time = 0 + time_use_list = [] + directory = Path('./') + filename_pattern = 'OpBasicInfo.csv' + + for file in directory.rglob(filename_pattern): + get_time(file, time_use_list) + + if len(time_use_list) > 0: + min_time = min(time_use_list) + + print(min_time) + +if __name__ == '__main__': + find_min_time() + + + diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..d34249c34 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import os +import sys +import numpy as np + +loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + + +def verify_result(output, golden): + real_result = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + return False + print("test pass") + return True + +if __name__ == '__main__': + + if os.path.exists(sys.argv[1]) and os.path.exists(sys.argv[2]): + verify_result(sys.argv[1], sys.argv[2]) + else: + print("[ERROR] result error") + exit(1) + diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..992759c95 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/common.cpp @@ -0,0 +1,80 @@ +/** + * @file common.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "common.h" + +#include +#include +#include + +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + size_t writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..481a31df8 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/main.cpp @@ -0,0 +1,162 @@ +/** + * @file main.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include +#include +#include + +#include +#include + +#include "acl/acl.h" +#include "common.h" +#include "op_runner.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..7e1623aed --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,453 @@ +/** + * @file op_runner.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "op_runner.h" + +#include +#include + +#include "acl/acl_op_compiler.h" +#include "aclnn_add_custom.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = + aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), nullptr, 0, + GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = + aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), nullptr, 0, + GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = + aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..da04cf6c9 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,51 @@ +/** + * @file operator_desc.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "operator_desc.h" + +#include "common.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/build.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..6fc77c881 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,73 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 1; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..323f3076f --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,22 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..44d6abf24 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,62 @@ +# set custom compile options +add_ops_compile_options(ALL OPTIONS -DASCENDC_DUMP) +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..07a286690 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,84 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + this->blockLength = totalLength / AscendC::GetBlockNum(); + this->tileNum = tileNum; + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * AscendC::GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * AscendC::GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * AscendC::GetBlockIdx(), this->blockLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); + AscendC::DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + AscendC::DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = inQueueY.DeQue(); + AscendC::LocalTensor zLocal = outQueueZ.AllocTensor(); + AscendC::Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + AscendC::LocalTensor zLocal = outQueueZ.DeQue(); + AscendC::DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX, inQueueY; + AscendC::TQue outQueueZ; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + AscendC::GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/README.md b/operator_contrib/CommonMistake/TimeOverrunBaseline/README.md new file mode 100644 index 000000000..8d69bcbbd --- /dev/null +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/README.md @@ -0,0 +1,20 @@ +## 样例说明 +该样例用于复现算子AclNN调用时出现test fail for performance exceeds baseline data + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` +time_base = 3720000 time_use = 13191839840 +test pass +test fail for performance exceeds baseline data + +``` +注：time_use根据硬件差异会有所不同，当time_use大于time_base时，就会出现test fail for performance exceeds baseline data + + +## 解决思路 + +1.可以对数据进行多核计算 +2.充分使用UB空间减少kernel侧循环次数 +3.避免频繁数据从GM到localTensor的数据搬运，避免频繁使用GetValue、SetValue等API \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/README.md b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/README.md new file mode 100644 index 000000000..75fa533f7 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/README.md @@ -0,0 +1,54 @@ +## 目录结构介绍 +``` +├── AclNNInvocation //通过aclnn调用的方式调用AddCustom算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子输入/输出个数，输入/输出大小等 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真实数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ └── run.sh // 执行命令脚本 +``` +## 代码实现介绍 +完成自定义算子的开发部署后，可以通过单算子调用的方式来验证单算子的功能。src/main.cpp代码为单算子API执行方式。单算子API执行是基于C语言的API执行算子，无需提供单算子描述文件进行离线模型的转换，直接调用单算子API接口。 + +自定义算子编译部署后，会自动生成单算子API，可以直接在应用程序中调用。算子API的形式一般定义为“两段式接口”，形如： + ```cpp + // 获取算子使用的workspace空间大小 + aclnnStatus aclnnAddCustomGetWorkspaceSize(const aclTensor *x, const aclTensor *y, const alcTensor *out, uint64_t workspaceSize, aclOpExecutor **executor); + // 执行算子 + aclnnStatus aclnnAddCustom(void *workspace, int64_t workspaceSize, aclOpExecutor **executor, aclrtStream stream); + ``` +其中aclnnAddCustomGetWorkspaceSize为第一段接口，主要用于计算本次API调用计算过程中需要多少的workspace内存。获取到本次API计算需要的workspace大小之后，按照workspaceSize大小申请Device侧内存，然后调用第二段接口aclnnAddCustom执行计算。具体参考[AscendCL单算子调用](https://hiascend.com/document/redirect/CannCommunityAscendCInVorkSingleOp)>单算子API执行 章节。 + +## 运行样例算子 +### 1. 编译算子工程 +运行此样例前，请参考[编译算子工程](../README.md#operatorcompile)完成前期准备。 +### 2. aclnn调用样例运行 + + - 进入到样例目录 + + ```bash + cd ${git_clone_path}/samples/operator/AddCustomSample/FrameworkLaunch/AclNNInvocation + ``` + - 样例执行 + + 样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + + ```bash + bash run.sh + ``` +## 更新说明 +| 时间 | 更新事项 | +| ---------- | ------------ | +| 2024/05/22 | 新增本readme | \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/common.h b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..11bb4aeca --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** + * @file common.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/op_runner.h b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..66b6ea2c3 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,180 @@ +/** + * @file op_runner.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "acl/acl.h" +#include "aclnn/acl_meta.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/operator_desc.h b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..6d8ee0905 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** + * @file operator_desc.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/run.sh b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/run.sh new file mode 100644 index 000000000..25433eb68 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while :; do + case "$1" in + # float16, float, int32 + -v | --dtype) + DTYPE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "[ERROR] Unexpected option: $1" + break + ;; + esac +done + +if [ -n "$ASCEND_INSTALL_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_INSTALL_PATH +elif [ -n "$ASCEND_HOME_PATH" ]; then + _ASCEND_INSTALL_PATH=$ASCEND_HOME_PATH +else + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + _ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + else + _ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $_ASCEND_INSTALL_PATH/bin/setenv.bash +export DDK_PATH=$_ASCEND_INSTALL_PATH +export NPU_HOST_LIB=$_ASCEND_INSTALL_PATH/lib64 + +function main { + # 1. 清除遗留生成文件和日志文件 + rm -rf $HOME/ascend/log/* + rm ./input/*.bin + rm ./output/*.bin + + # 2. 生成输入数据和真值数据 + cd $CURRENT_DIR + python3 scripts/gen_data.py + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译acl可执行文件 + cd $CURRENT_DIR + rm -rf build + mkdir -p build + cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_add_op + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output_z.bin output/golden.bin + if [ $? -ne 0 ]; then + echo "ERROR: verify result failed!" + return 1 + fi +} + +main diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/acl.json b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/gen_data.py b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..ea557313c --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os + +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float32) + input_y = np.random.uniform(1, 100, [8, 2048]).astype(np.float32) + golden = (input_x + input_y).astype(np.float32) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/verify_result.py b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..98cab9c85 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-4 +absolute_tol = 1e-5 +error_tol = 1e-4 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float32).reshape(-1) + golden = np.fromfile(golden, dtype=np.float32).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 0: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..730ba0013 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,67 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_add_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_add_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_add_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..992759c95 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/common.cpp @@ -0,0 +1,80 @@ +/** + * @file common.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "common.h" + +#include +#include +#include + +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + size_t writeSize = write(fd, buffer, size); + (void)close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/main.cpp b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..289107257 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/main.cpp @@ -0,0 +1,162 @@ +/** + * @file main.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include +#include +#include + +#include +#include + +#include "acl/acl.h" +#include "common.h" +#include "op_runner.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{8, 2048}; + aclDataType dataType = ACL_FLOAT; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape.size(), shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_z.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/op_runner.cpp b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..7e1623aed --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,453 @@ +/** + * @file op_runner.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "op_runner.h" + +#include +#include + +#include "acl/acl_op_compiler.h" +#include "aclnn_add_custom.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + (void)aclDestroyTensor(inputTensor_[i]); + (void)aclDestroyDataBuffer(inputBuffers_[i]); + (void)aclrtFree(devInputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostInputs_[i]); + } else { + (void)aclrtFreeHost(hostInputs_[i]); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + (void)aclDestroyTensor(outputTensor_[i]); + (void)aclDestroyDataBuffer(outputBuffers_[i]); + (void)aclrtFree(devOutputs_[i]); + if (g_isDevice) { + (void)aclrtFree(hostOutputs_[i]); + } else { + (void)aclrtFreeHost(hostOutputs_[i]); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = + aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), nullptr, 0, + GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = + aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), nullptr, 0, + GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + auto ret = + aclnnAddCustomGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustomGetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + + ret = aclnnAddCustom(workspace, workspaceSize, handle, stream); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute aclnnAddCustom success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + +template void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..da04cf6c9 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,51 @@ +/** + * @file operator_desc.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "operator_desc.h" + +#include "common.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakeLists.txt b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakeLists.txt new file mode 100755 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakePresets.json b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakePresets.json new file mode 100644 index 000000000..ba45da377 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/build.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/build.sh new file mode 100755 index 000000000..4be96d7d8 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') +if [ "$cmake_version" \< "3.19.0" ] ; then + opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) + echo $opts + cmake .. $opts +else + cmake .. --preset=default +fi +target=package +if [ "$1"x != ""x ]; then target=$1; fi + +cmake --build . --target $target -j16 +if [ $? -ne 0 ]; then exit 1; fi + +if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target $target -j16 + fi +fi + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/config.cmake b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/func.cmake b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/func.cmake new file mode 100755 index 000000000..ad187e7d6 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/func.cmake @@ -0,0 +1,228 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_ops_replay_targets) + cmake_parse_arguments(OPREPLAY "" "OPS_INFO;COMPUTE_UNIT;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + # ccec compile options + set(ccec_base_opts -c -O2 --cce-aicore-only -mllvm -cce-aicore-function-stack-size=16000 + -mllvm -cce-aicore-record-overflow=false -std=c++17) + set(ccec_extopts_ascend310p --cce-aicore-arch=dav-m200 -mllvm -cce-aicore-fp-ceiling=2) + set(ccec_extopts_ascend910 --cce-aicore-arch=dav-c100) + set(ccec_extopts_ascend910b --cce-aicore-arch=dav-c220-cube) + file(MAKE_DIRECTORY ${OPREPLAY_OUT_DIR}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_replay_build.py + ${OPREPLAY_OPS_INFO} + "${OPREPLAY_OPS_BATCH}" "${OPREPLAY_OPS_ITERATE}" + ${OPREPLAY_IMPL_DIR} + ${OPREPLAY_OUT_DIR} + ${OPREPLAY_COMPUTE_UNIT} + ) + file(GLOB replay_kernel_entries ${OPREPLAY_OUT_DIR}/*.cce) + if (NOT "${replay_kernel_entries}x" STREQUAL "x") + foreach(replay_kernel_file ${replay_kernel_entries}) + get_filename_component(replay_kernel_file_name "${replay_kernel_file}" NAME) + string(REPLACE "_entry.cce" "" op_kerne_name ${replay_kernel_file_name}) + file(GLOB replay_lib_src ${OPREPLAY_OUT_DIR}/${op_kerne_name}*.cpp) + set(OP_TILING_DATA_H_PATH ${OPREPLAY_OUT_DIR}/${op_kerne_name}_tiling_data.h) + add_library(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} SHARED ${replay_lib_src}) + if(EXISTS ${OP_TILING_DATA_H_PATH}) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -include ${OP_TILING_DATA_H_PATH} + ) + endif() + target_compile_definitions(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + ${op_kerne_name}=${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + ) + target_compile_options(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE + -D__ASCENDC_REPLAY__ + ) + target_link_libraries(replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} PRIVATE intf_pub + tikreplaylib::${OPREPLAY_COMPUTE_UNIT} + register + ) + add_custom_command(OUTPUT ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + COMMAND ccec ${ccec_base_opts} ${ccec_extopts_${OPREPLAY_COMPUTE_UNIT}} ${replay_kernel_file} + -o ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DEPENDS ${replay_kernel_file} + ) + add_custom_target(replay_kernel_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} ALL + DEPENDS ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + ) + install(TARGETS replay_${op_kerne_name}_${OPREPLAY_COMPUTE_UNIT} + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + install(FILES ${OPREPLAY_OUT_DIR}/${op_kerne_name}_entry_${OPREPLAY_COMPUTE_UNIT}.o + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_replay + ) + endforeach() + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) +endfunction() diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/intf.cmake b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/intf.cmake new file mode 100755 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/makeself.cmake b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/makeself.cmake new file mode 100755 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/__init__.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_bin_param_build.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..decf34544 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + atype = atype.replace('list', 'list_') + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_impl_build.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..7fe177da1 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_impl_build.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import stat +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int8_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]} + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + for x in __inputs__ + __outputs__: + x_n = x.get("param_name").upper() + x_fmt = x.get("format") + x_dtype = x.get("dtype") + options.append("-DDTYPE_{n}={t}".format(n=x_n, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{n}={ot}".format(n=x_n, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{n}=FORMAT_{f}".format(n=x_n, f=x_fmt)) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}") +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + ccec = os.environ.get('CCEC_REAL_PATH') + if ccec is None: + ccec = shutil.which("ccec") + if ccec != None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = os.path.join(PYF_PATH, "..", "ascendc", ascendc_src_dir, ascendc_src_file) + if not os.path.exists(src): + src = os.path.join(PYF_PATH, ascendc_src_file) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is {}") + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}]) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'listInt': '[]', + 'listFloat': '[]', 'listBool': '[]', 'listListInt': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsname = [] + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + att_type = att_type.replace('LIST', 'LIST_') + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, kern_name, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_ops_config.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..7a97180be --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_ops_config.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {"MIX": 0, "AiCore": 1, "VectorCore": 2} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, os.path.join(soc, op_dir, bin_file_name), config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_replay_build.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/batch_replay_impl.temp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/batch_replay_impl.temp new file mode 100755 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/code_channel_infer.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..137c9f39f --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/code_channel_infer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for ccec cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC + if params.compile_options is None: + compile_options = [] + else: + compile_options = params.compile_options + ccec = shutil.which("ccec") + if ccec is not None: + ccec_path = os.path.dirname(ccec) + tikcpp_path = os.path.realpath(os.path.join(ccec_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + compile_options.append("-I" + tikcpp_path) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + compile_options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + compile_options += ["-include", params.tiling_header] + arch = "dav-c220-cube" + sub_core_type = "AIC" + optional_core = "AiCore" + compile_cmd = [shutil.which("ccec"), '-c', '-O3'] + compile_cmd += compile_options + temp_file_name_tag = "_" + str(os.getpid()) + "_temp.o" + dst_file = os.path.join(kernel_meta_dir, kernel_name + temp_file_name_tag) + compile_cmd += [params.src_file, "--cce-aicore-arch={}".format(arch), + "--cce-aicore-only", "-o", dst_file, + "-mllvm", "-cce-aicore-function-stack-size=16000", + "-mllvm", "-cce-aicore-record-overflow=true", + "-mllvm", "-cce-aicore-addr-transform"] + compile_cmd += ["-std=c++17"] + print('get_code_channel: ', ' '.join(compile_cmd)) + proc = subprocess.Popen( + compile_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel coretype compile error: ', out.decode()) + msg = "compile %s error :%s\n" % (params.src_file, out.decode()) + raise Exception(f"get_code_channel coretype error, msg is{msg}") + objdump_cmd = ['objdump', '-s', '-j', '.text', '{}'.format(dst_file)] + + proc = subprocess.Popen( + objdump_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + if proc.returncode != 0: + print('get_code_channel objdump error: ', out.decode()) + msg = "get_code_channel objdump %s error :%s\n" % (src_file, out.decode()) + raise Exception(f"get_code_channel objdump error, msg is{msg}") + os.remove(dst_file) + lines = out.decode('utf-8').split('\n') + for line in lines: + insts = line.strip().split() + if len(insts) < 5: + continue + for inst in insts[1:]: + if len(inst) != 8: + continue + if inst[6] == 'f': + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/const_var.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/const_var.py new file mode 100755 index 000000000..8b32c3b91 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..55e12e5ed --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,57 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy ai_core operators implements +# tbe_impl_files_num=$(ls $project_path/tbe/impl/* 2> /dev/null | wc -l) +# if [[ "$tbe_impl_files_num" -gt 0 ]];then +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/ai_core/tbe/customize_impl +# cp -r ${project_path}/tbe/impl/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/vector_core/tbe/customize_impl +# fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi + +# merge aicpu.ini and aicore.ini to generate npu_supported_ops.json +# mkdir -p ${build_path}/framework/op_info_cfg +# mkdir -p ${build_path}/framework/op_info_cfg/aicpu_kernel +# mkdir -p ${build_path}/framework/op_info_cfg/ai_core + +# if [[ -d "${project_path}/tbe/op_info_cfg/ai_core" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/tbe/op_info_cfg/ai_core ${build_path}/framework/op_info_cfg/ai_core +# fi + +# if [[ -d "${project_path}/cpukernel/op_info_cfg/aicpu_kernel" ]]; then +# bash ${project_path}/cmake/util/gen_ops_filter.sh ${project_path}/cpukernel/op_info_cfg/aicpu_kernel ${build_path}/framework/op_info_cfg/aicpu_kernel +# fi + +# aicpu_filter_file=${build_path}/framework/op_info_cfg/aicpu_kernel/npu_supported_ops.json +# aicore_filter_file=${build_path}/framework/op_info_cfg/ai_core/npu_supported_ops.json +# if [[ -f "${aicpu_filter_file}" ]] && [[ ! -f "${aicore_filter_file}" ]]; then +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi +# if [[ -f "${aicore_filter_file}" ]] && [[ ! -f "${aicpu_filter_file}" ]]; then +# cp $aicore_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + +# if [[ -f "${aicore_filter_file}" ]] && [[ -f "${aicpu_filter_file}" ]]; then +# chmod u+w ${aicpu_filter_file} +# python3 ${project_path}/cmake/util/insert_op_info.py ${aicore_filter_file} ${aicpu_filter_file} +# chmod u-w ${aicpu_filter_file} +# cp $aicpu_filter_file ${build_path}/makepkg/packages/vendors/$vendor_name/framework/tensorflow +# fi + diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_ops_filter.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_version_info.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_op_info.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_simplified_keys.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..ace727b90 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/insert_simplified_keys.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_entry.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_impl.temp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_impl.temp new file mode 100755 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/COPYING b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/COPYING new file mode 100755 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/README.md b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/README.md new file mode 100755 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/VERSION b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/VERSION new file mode 100755 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/make-release.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself-header.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.1 b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.lsm b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.lsm new file mode 100755 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/run-tests.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/opdesc_parser.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..c38e79edc --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/opdesc_parser.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_name.append(self._parse_str(conf)) + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_name.append(self._parse_str(conf)) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._parse_str(conf) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/parse_ini_to_json.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..721465fee --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # Only the owner and group have rights + os.chmod(json_file_real_path, stat.S_IWGRP + stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/preset_parse.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/preset_parse.py new file mode 100755 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_codegen.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_impl.temp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_impl.temp new file mode 100755 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/tiling_data_def_build.py b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..a96304261 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/cmake/util/tiling_data_def_build.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' memcpy(const_data, tiling, sizeof({}));\n'.format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/CMakeLists.txt b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/CMakeLists.txt new file mode 100755 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/CMakeLists.txt b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc new file mode 100644 index 000000000..2cd837ce5 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/framework/tf_plugin/tensorflow_add_custom_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("AddCustom") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Add") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/CMakeLists.txt b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/CMakeLists.txt new file mode 100755 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp new file mode 100644 index 000000000..6fc77c881 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp @@ -0,0 +1,73 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "add_custom_tiling.h" +#include "register/op_def_registry.h" + +namespace optiling { +const uint32_t BLOCK_DIM = 1; +const uint32_t TILE_NUM = 8; +static ge::graphStatus TilingFunc(gert::TilingContext *context) +{ + TilingData tiling; + uint32_t totalLength = context->GetInputShape(0)->GetOriginShape().GetShapeSize(); + context->SetBlockDim(BLOCK_DIM); + tiling.set_totalLength(totalLength); + tiling.set_tileNum(TILE_NUM); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} // namespace optiling + +namespace ge { +static graphStatus InferShape(gert::InferShapeContext *context) +{ + const gert::Shape *x1_shape = context->GetInputShape(0); + gert::Shape *y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} + +static graphStatus InferDataType(gert::InferDataTypeContext *context) +{ + const auto inputDataType = context->GetInputDataType(0); + context->SetOutputDataType(0, inputDataType); + return ge::GRAPH_SUCCESS; +} +} // namespace ge + +namespace ops { +class AddCustom : public OpDef { +public: + explicit AddCustom(const char *name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape).SetInferDataType(ge::InferDataType); + this->AICore() + .SetTiling(optiling::TilingFunc) + .AddConfig("ascend310b"); + } +}; +OP_ADD(AddCustom); +} // namespace ops diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h new file mode 100644 index 000000000..323f3076f --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h @@ -0,0 +1,22 @@ +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(TilingData) +TILING_DATA_FIELD_DEF(uint32_t, totalLength); +TILING_DATA_FIELD_DEF(uint32_t, tileNum); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(AddCustom, TilingData) +} // namespace optiling +#endif // ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/CMakeLists.txt b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/CMakeLists.txt new file mode 100755 index 000000000..0d31a444c --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/CMakeLists.txt @@ -0,0 +1,61 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp new file mode 100644 index 000000000..584092996 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp @@ -0,0 +1,96 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelAdd { +public: + __aicore__ inline KernelAdd() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength, uint32_t tileNum) + { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + this->blockLength = totalLength / GetBlockNum(); + this->tileNum = tileNum; + ASSERT(tileNum != 0 && "tile num can not be zero!"); + this->tileLength = this->blockLength / tileNum / BUFFER_NUM; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); + } + __aicore__ inline void Process() + { + int32_t loopCount = this->tileNum * BUFFER_NUM; + for (int32_t i = 0; i < loopCount; i++) { + CopyIn(i); + Compute(i); + CopyOut(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + LocalTensor yLocal = inQueueY.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileLength], this->tileLength); + DataCopy(yLocal, yGm[progress * this->tileLength], this->tileLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + LocalTensor yLocal = inQueueY.DeQue(); + LocalTensor zLocal = outQueueZ.AllocTensor(); + Add(zLocal, xLocal, yLocal, this->tileLength); + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(int32_t progress) + { + LocalTensor zLocal = outQueueZ.DeQue(); + DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); + outQueueZ.FreeTensor(zLocal); + } + +private: + TPipe pipe; + TQue inQueueX, inQueueY; + TQue outQueueZ; + GlobalTensor xGm; + GlobalTensor yGm; + GlobalTensor zGm; + uint32_t blockLength; + uint32_t tileNum; + uint32_t tileLength; +}; + +extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); + KernelAdd op; + op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); + op.Process(); +} + +#ifndef ASCENDC_CPU_DEBUG +// call of kernel function +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, + uint8_t *workspace, uint8_t *tiling) +{ + add_custom<<>>(x, y, z, workspace, tiling); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/install.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/install.sh new file mode 100755 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/upgrade.sh b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/upgrade.sh new file mode 100755 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/CommonMistake/TypeDeletion/README.md b/operator_contrib/CommonMistake/TypeDeletion/README.md new file mode 100644 index 000000000..d83ad03ac --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/README.md @@ -0,0 +1,34 @@ +## 样例说明 +该样例用于复现算子实现支持数据类型缺失的错误 + +## 测试说明 + +编译部署AddCustom后，运行AclNNInvocation调用，即可复现错误，界面会输出 + ``` + +INFO: execute op! +[INFO] Set device[0] success +[INFO] Get RunMode[0] success +[INFO] Init resource success +[INFO] Set input success +[INFO] Copy input[0] success +[INFO] Copy input[1] success +[INFO] Create stream success +[ERROR] Get Operator Workspace failed. error code is 561003 +[ERROR] Run op failed +[INFO] Reset Device success +[INFO] Destory resource success + + +``` + +使用grep在plog日志查找ERROR关键字有类似信息 + +[ERROR] OP(3462352,execute_add_op):2024-08-14-09:45:07.078.391 [nnopbase_executor.cpp:1383][NNOP][NnopbaseExecutorCheckSocVersionAndParam][3462352] errno[561003] OpName:[AddCustom_0] **The binary bin corresponding to dtype and format is not found！** + + + + +## 解决思路 +1、阅读代码，检查算子原型注册时定义的数据类型、数据排布格式是否和测试程序内给出的测试数据匹配。 + diff --git a/operator_contrib/CommonMistake/TypeDeletion/run_test.sh b/operator_contrib/CommonMistake/TypeDeletion/run_test.sh new file mode 100755 index 000000000..1de7c6039 --- /dev/null +++ b/operator_contrib/CommonMistake/TypeDeletion/run_test.sh @@ -0,0 +1,4 @@ +rm -rf ~/ascend/log/* +cd AddCustom;./build.sh;./build_out/*.run +cd ../AclNNInvocation;bash run.sh +cd ~/ascend/log; grep -rn errorStr \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/common.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/op_runner.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/operator_desc.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..17607353d --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,60 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + bool keepDims; + bool ignoreNan; + char *dtype; + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/run.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/run.sh new file mode 100644 index 000000000..98c688714 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/run.sh @@ -0,0 +1,109 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + # rm ./input/*.bin + rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_op >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output.bin output/golden.bin + +} + +main diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/acl.json b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/gen_data.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..f339e299f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,24 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os +np.random.seed(0) +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 65504 / 2048, [2048]).astype(np.float16) + golden = input_x * np.sqrt(2) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/verify_result.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..3b5751e1f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "first 10 mistakes data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 10: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..ee73e930f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include + ${BUILT_PKG_PATH}/include/aclnnop +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/main.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..081e72b84 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/main.cpp @@ -0,0 +1,163 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{2048}; + std::vector shape_out{2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape_out.size(), shape_out.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..2934f4cae --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,485 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_demo.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto ret = aclDestroyTensor(inputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(inputBuffers_[i]); + + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devInputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostInputs_[i]); + } else { + ret = aclrtFreeHost(hostInputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto ret = aclDestroyTensor(outputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(outputBuffers_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devOutputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostOutputs_[i]); + } else { + ret = aclrtFreeHost(hostOutputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + + auto ret = aclnnDemoGetWorkspaceSize(inputTensor_[0], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + ret = aclnnDemo(workspace, workspaceSize, handle, stream); + + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute Operator success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo.json b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo.json new file mode 100644 index 000000000..bc7789518 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo.json @@ -0,0 +1,22 @@ +[ + { + "op": "Demo", + "language": "cpp", + "input_desc": [ + { + "name": "x", + "param_type": "required", + "format": ["ND"], + "type": ["float16"] + } + ], + "output_desc": [ + { + "name": "y", + "param_type": "required", + "format": ["ND"], + "type": ["float16"] + } + ] + } +] \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakeLists.txt new file mode 100644 index 000000000..ea2c509e5 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakeLists.txt @@ -0,0 +1,74 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + if(CMAKE_CROSS_LIBRARY_PATH) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + endif() + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakePresets.json b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakePresets.json new file mode 100644 index 000000000..d32bf6d9a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b;ascend910b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/build.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/build.sh new file mode 100755 index 000000000..113fa60d8 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/build.sh @@ -0,0 +1,95 @@ +#!/bin/bash +if [ -z "$BASE_LIBS_PATH" ]; then + if [ -z "$ASCEND_HOME_PATH" ]; then + if [ -z "$ASCEND_AICPU_PATH" ]; then + echo "please set env." + exit 1 + else + export ASCEND_HOME_PATH=$ASCEND_AICPU_PATH + fi + else + export ASCEND_HOME_PATH=$ASCEND_HOME_PATH + fi +else + export ASCEND_HOME_PATH=$BASE_LIBS_PATH +fi +echo "using ASCEND_HOME_PATH: $ASCEND_HOME_PATH" +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/config.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/config.cmake new file mode 100755 index 000000000..72690b9e4 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/config.cmake @@ -0,0 +1,30 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_AUTOGEN_GROUPPROTO_PATH ${CMAKE_BINARY_DIR}/autogen/group_proto) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH} ${ASCEND_AUTOGEN_GROUPPROTO_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + ) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/func.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/func.cmake new file mode 100755 index 000000000..c4da5fee8 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/func.cmake @@ -0,0 +1,212 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + + set(ENV{ENABLE_SOURCE_PACAKGE} ${OPBUILD_ENABLE_SOURCE}) + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + unset(ENV{ENABLE_SOURCE_PACAKGE}) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_gen_options.py + ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} ${OP_TYPE} ${OP_COMPILE_COMPUTE_UNIT} + ${OP_COMPILE_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR) + if (${EXEC_RESULT}) + message("add ops compile options info: ${EXEC_INFO}") + message("add ops compile options error: ${EXEC_ERROR}") + message(FATAL_ERROR "add ops compile options failed!") + endif() +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + --opc-config-file ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + # add Environment Variable Configurations of python & ccache + set(_ASCENDC_ENV_VAR) + list(APPEND _ASCENDC_ENV_VAR export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} &&) + # whether need judging CMAKE_C_COMPILER_LAUNCHER + if(${CMAKE_CXX_COMPILER_LAUNCHER} MATCHES "ccache$") + list(APPEND _ASCENDC_ENV_VAR export ASCENDC_CCACHE_EXECUTABLE=${CMAKE_CXX_COMPILER_LAUNCHER} &&) + endif() + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND ${_ASCENDC_ENV_VAR} bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/intf.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/intf.cmake new file mode 100755 index 000000000..d2643bbc7 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/intf.cmake @@ -0,0 +1,28 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include + ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel +) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/makeself.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/makeself.cmake new file mode 100755 index 000000000..2c482b042 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/makeself.cmake @@ -0,0 +1,33 @@ +execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) + +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() + +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + +if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") + execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + + execute_process( + COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) +endif() + + diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/__init__.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_bin_param_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..39bfbb13c --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import argparse +import sys +import os +import json +import hashlib +import re +import copy +from collections import defaultdict +from typing import Dict, List, Set, Tuple, NamedTuple + +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ParamInfo(NamedTuple): + dtype_list: list + format_list: list + dtype_for_bin_list: dict + format_for_bin_list: dict + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + self.tiling_keys = set() + self.op_debug_config = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def set_tiling_key(self: any, tiling_key_info: Set): + if tiling_key_info: + self.tiling_keys.update(tiling_key_info) + + def set_op_debug_config(self: any, op_debug_config: str): + if op_debug_config: + self.op_debug_config = op_debug_config + + def get_full_list(self: any): + dtype_list = [] + for dtype_in in self.input_dtype: + dtype_list.append(dtype_in.split(',')) + for dtype_out in self.output_dtype: + dtype_list.append(dtype_out.split(',')) + + format_list = [] + for fmt_in in self.input_fmt: + format_list.append(fmt_in.split(',')) + for fmt_out in self.output_fmt: + format_list.append(fmt_out.split(',')) + + dtype_for_bin_list = [[] for _ in range(len(self.input_dtype) + len(self.output_dtype))] + format_for_bin_list = copy.deepcopy(dtype_for_bin_list) + + for key, value in self.input_dtype_for_bin.items(): + dtype_for_bin_list[key] = value.split(',') + for key, value in self.output_dtype_for_bin.items(): + dtype_for_bin_list[key + len(self.input_dtype)] = value.split(',') + for key, value in self.input_fmt_for_bin.items(): + format_for_bin_list[key] = value.split(',') + for key, value in self.output_fmt_for_bin.items(): + format_for_bin_list[key + len(self.input_dtype)] = value.split(',') + + return ParamInfo(dtype_list, format_list, dtype_for_bin_list, format_for_bin_list) + + + def gen_bin_cprs_list(self: any, param_info: ParamInfo): + combine_dict = {} + origin_combine_dict = {} + for cob_idx in range(0, len(self.input_dtype[0].split(','))): + origin_combine = "" + combine = "" + for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): + if (param_info.dtype_for_bin_list[param_idx]): + combine += param_info.dtype_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.dtype_list[param_idx][cob_idx] + origin_combine += param_info.dtype_list[param_idx][cob_idx] + if (param_info.format_for_bin_list[param_idx]): + combine += param_info.format_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.format_list[param_idx][cob_idx] + origin_combine += param_info.format_list[param_idx][cob_idx] + if (combine not in combine_dict): + combine_dict[combine] = [] + combine_dict[combine].append(cob_idx) + origin_combine_dict[origin_combine] = cob_idx + for key, value in combine_dict.items(): + if (key not in origin_combine_dict): + print(f"WARNING: ForBinQuery {key} not in origin combine") + self.bin_save_list += value + continue + if len(value) == 1 and value[0] == origin_combine_dict[key]: + self.bin_save_list += value + continue + self.bin_cprs_head.append(origin_combine_dict[key]) + self.bin_cprs_list.append(value) + for index, sub_list in enumerate(self.bin_cprs_list): + if self.bin_cprs_head[index] not in self.bin_save_list: + continue + sub_list.append(self.bin_cprs_head[index]) + self.bin_save_list += self.bin_cprs_head + + + def gen_for_bin_list(self: any, param_info: ParamInfo): + combine_size = len(self.input_dtype[0].split(',')) + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + + self.input_dtype_for_bin_list = [[] for _ in range(input_size)] + self.output_dtype_for_bin_list = [[] for _ in range(output_size)] + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) + self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) + + for index, sub_list in enumerate(self.bin_cprs_list): + head_idx = self.bin_cprs_head[index] + for cmb_idx in sub_list: + for i in range(0, input_size): + self.input_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i][cmb_idx]) + self.input_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i][cmb_idx]) + for i in range(0, output_size): + self.output_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i + input_size][cmb_idx]) + self.output_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i + input_size][cmb_idx]) + + + def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.input_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.input_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.input_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.input_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.input_fmt[i] = new_format_str[:-1] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.output_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.output_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.output_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i + input_size]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.output_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i + input_size]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.output_fmt[i] = new_format_str[:-1] + + + def is_set_for_bin_query(self: any): + return any([ + self.input_dtype_for_bin, + self.output_dtype_for_bin, + self.input_fmt_for_bin, + self.output_fmt_for_bin, + ]) + + + def for_bin_list_match(self: any): + if not self.is_set_for_bin_query(): + return + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + param_info = self.get_full_list() + self.gen_bin_cprs_list(param_info) + self.gen_for_bin_list(param_info) + if len(self.bin_save_list) == len(self.input_dtype[0].split(',')): + print(f'WARNING: ForBinQuery can not compress number of bin file with this set, please check!!.') + return + self.rm_cprs_cmb(param_info.dtype_list, param_info.format_list, input_size, output_size) + + + def gen_input_json(self: any, auto_gen_path: str): + key_map = {} + self.for_bin_list_match() + count = len(self.input_dtype[0].split(',')) + required_parameters = set() + index_value = -1 + + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + required_parameter = [] + op_node = {} + + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + if self.is_set_for_bin_query() and self.input_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.input_dtype_for_bin_list[idx][i] + para['format'] = ifmts[i] + if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.input_fmt_for_bin_list[idx][i] + para['paramType'] = itype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + + input_parameter_key = (idtypes[i], ifmts[i]) + if itype == 'dynamic': + inputs.append([para]) + required_parameter.append(input_parameter_key) + elif itype == 'required': + inputs.append(para) + required_parameter.append(input_parameter_key) + else: + inputs.append(para) + + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + if self.is_set_for_bin_query() and self.output_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.output_dtype_for_bin_list[idx][i] + para['format'] = ofmts[i] + if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.output_fmt_for_bin_list[idx][i] + para['paramType'] = otype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + output_parameter_key = (odtypes[i], ofmts[i]) + if otype == 'dynamic': + outputs.append([para]) + required_parameter.append(output_parameter_key) + elif otype == 'required': + outputs.append(para) + required_parameter.append(output_parameter_key) + else: + outputs.append(para) + + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + + required_parameter_tuple = tuple(required_parameter) + if required_parameter_tuple in required_parameters: + continue + else: + required_parameters.add(required_parameter_tuple) + index_value +=1 + + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) + + + def _write_build_cmd(self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str): + hard_soc = const_var.conv_soc_ver(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + + bin_cmd_str = 'res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ + --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic ' + + build_cmd_var = "#!/bin/bash\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' + plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") + plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") + if plog_level is None: + build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR + if plog_stdout is None: + build_cmd_var += const_var.SET_PLOG_STDOUT + build_cmd_var += const_var.SRC_ENV + if hard_soc == "Ascend610Lite": + build_cmd_var += f'export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n' + build_cmd_var += bin_cmd_str.format(fun=self.op_intf, soc=hard_soc, param=param_file, + impl='high_performance,optional') + enable_tiling_keys = False + if self.tiling_keys: + tiling_keys_list = sorted(list(self.tiling_keys)) + tiling_key_str = ','.join([str(_key) for _key in tiling_keys_list]) + build_cmd_var += f' --tiling_key="{tiling_key_str}"' + enable_tiling_keys = True + + if self.op_debug_config: + op_debug_str = ','.join([str(_key) for _key in list(self.op_debug_config)]) + build_cmd_var += f' --op_debug_config={op_debug_str}' + + build_cmd_var += ")\n" + build_cmd_var += "\n" + if enable_tiling_keys is False: + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + else: + build_cmd_var += "if [ $? -eq 1 ]; then\n" + build_cmd_var += " if echo \"${res}\" | \ +grep -q \"None of the given tiling keys are in the supported list\"; then\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " else\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " exit 1\n" + build_cmd_var += " fi\n" + build_cmd_var += "else\n" + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + build_cmd_var += "fi\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' + + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write(build_cmd_var) + + +def get_tiling_keys(tiling_keys: str) -> Set: + all_tiling_keys = set() + if not tiling_keys: + return all_tiling_keys + + tiling_key_list = tiling_keys.split(';') + for tiling_key_value in tiling_key_list: + pattern = r"(? int(end): + continue + for i in range(int(start), int(end) + 1): + all_tiling_keys.add(i) + elif tiling_key_value.isdigit(): + all_tiling_keys.add(int(tiling_key_value)) + return all_tiling_keys + + +def trans_soc_verion(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: + return low_soc_ver + return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: + tiling_key_info = defaultdict(set) + op_debug_config = defaultdict(set) + if not opc_config_file: + return tiling_key_info, op_debug_config + + if not os.path.exists(opc_config_file): + return tiling_key_info, op_debug_config + + with open(opc_config_file, 'r') as file: + contents = file.readlines() + + for _content in contents: + content = _content.strip() + opc_configs = content.split('@') + if len(opc_configs) < 3: + continue + + op_type = opc_configs[0] + if not op_type: + continue + + compute_unit = opc_configs[1] + if compute_unit: + compute_unit_list = compute_unit.split(';') + soc_lists = [] + for soc_ver in compute_unit_list: + short_soc_ver = trans_soc_verion(soc_ver) + soc_lists.append(short_soc_ver) + if soc not in soc_lists: + continue + + for options in opc_configs[2:]: + if "--tiling_key" in options: + format_tiling_keys = get_tiling_keys(options.split('=')[1]) + if format_tiling_keys: + tiling_key_info[op_type].update(format_tiling_keys) + if "--op_debug_config" in options: + format_debug_config = set(options.split("=")[1].split(';')) + if format_debug_config: + op_debug_config[op_type].update(format_debug_config) + + return tiling_key_info, op_debug_config + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str, + opc_config_file: str = '', ops: list = None): + if not os.path.exists(cfgfile): + print(f'INFO: {cfgfile} does not exists in this project, skip generating compile commands.') + return + + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) + tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) + auto_gen_path_dir = os.path.dirname(cfgfile) + all_soc_key = "ALL" + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + if op_desc.op_type in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) + if all_soc_key in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[all_soc_key]) + if op_desc.op_type in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) + if all_soc_key in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[all_soc_key]) + op_desc.gen_input_json(auto_gen_path_dir) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opc-config-file', nargs='?', const='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + if len(args.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(args.argv[1], + args.argv[2], + args.argv[3], + opc_config_file=args.opc_config_file) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_gen_options.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_gen_options.py new file mode 100755 index 000000000..2dd8ee047 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_gen_options.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import sys +import stat +import os +import re +import json +import const_var + + +def write_options_to_file(file_name: str, options_str: str, \ + op_type: str, compute_unit: str, split_char: str): + flags = os.O_WRONLY | os.O_CREAT + modes = stat.S_IWUSR | stat.S_IRUSR + try: + with os.fdopen(os.open(file_name, flags, modes), 'a') as fd: + fd.write(op_type + split_char + compute_unit + split_char + options_str + '\n') + except Exception as err: + print("write compile options config file failed") + raise(err) + + +def gen_compile_options(compile_options_file: str, op_type: str, \ + compute_unit: str, compile_options: list): + base_dir = os.path.dirname(compile_options_file) + opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") + compile_opt = [] + opc_debug_config = [] + opc_tiling_keys = "" + for opts in compile_options: + if "oom" in opts: + if opts == "--oom": + opc_debug_config.append("oom") + else: + raise RuntimeError(f"Unknown oom option format {opts}") + elif "--save-temp-files" in opts: + opc_debug_config.append("dump_cce") + elif "--tiling_key" in opts: + keys = opts.strip().split('=')[1].split(',') + keys_str = ";".join([key for key in keys]) + opc_tiling_keys = keys_str + else: + compile_opt.append(opts) + if len(compile_opt) > 0: + options_str = ';'.join([opt for opt in compile_opt]) + write_options_to_file(compile_options_file, options_str, op_type, compute_unit, ",") + opc_config_str = "" + if opc_debug_config: + opc_config_str = "--op_debug_config=" + ';'.join([opt for opt in opc_debug_config]) + if len(opc_tiling_keys) > 0: + if opc_config_str != "": + opc_config_str += "@" + opc_config_str += "--tiling_key=" + opc_tiling_keys + + if opc_config_str != "": + write_options_to_file(opc_config_file, opc_config_str, op_type, compute_unit, "@") + + +if __name__ == '__main__': + if len(sys.argv) < 4: + raise RuntimeError('arguments must greater than 4') + compute_soc = "" + comp_options = [] + for i in range(len(sys.argv) - 3): + if sys.argv[i + 3].upper().startswith("ASCEND"): + compute_soc += sys.argv[i + 3] + ";" + else: + comp_options.append(sys.argv[i + 3]) + if compute_soc != "": + compute_soc = compute_soc[0:-1] + gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_impl_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..caa3ab149 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_impl_build.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" + +import argparse +import glob +import sys +import os +import re +import datetime +from typing import List + +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = '''#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. +""" + +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex32": ["DT_COMPLEX32", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) + raise RuntimeError("cannot open %s" %(so_path)) + else: + msg = "load so succ " + so_path + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options.extend(compile_option_list[shortsoc]) + if '__ALLSOC__' in compile_option_list: + compile_options.extend(compile_option_list['__ALLSOC__']) + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): +{} + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = os.environ.get('BISHENG_REAL_PATH') + if bisheng is None: + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_current_build_config("enable_deterministic_mode") == 1: + options.append("-DDETERMINISTIC_MODE=1") + else: + options.append("-DDETERMINISTIC_MODE=0") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + msg = "start replay Acend C Operator {}, kernel name is {}" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API_BUILT_IN = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + + op_compile_option = '{}' + opp_path = os.environ.get('ASCEND_OPP_PATH') + dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) + if opp_path and os.path.exists(dat_path): + # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op + abs_rel_kernel_src_path = "{}" + extend_options = {{}} + extend_options['opp_kernel_hidden_dat_path'] = dat_path + compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ + extend_options) + else: + raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code == 0: + sup = "True" + reason = "" + elif err_code == 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + snake_case += "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + snake_case += "_" + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all: list = None): + self._build_paradefault() + if os.environ.get('BUILD_BUILTIN_OPP') != '1' and impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + print(f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check.") + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd, impl_path) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self: any, op_compile_option_all: list = None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format('OPTION', att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_virtual(self: any) -> str: + virt_exp = [] + for index in range(len(self.input_name)): + if self.input_virt.get(index) is None: + continue + val = [] + val.append('"param_name":"{}"'.format(self.input_name[index])) + val.append('"index":{}'.format(index)) + val.append('"dtype":"{}"'.format(self.input_dtype[index].split(',')[0])) + val.append('"format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"paramType":"optional"') + val.append('"shape":[1]') + val.append('"ori_shape":[1]') + virt_exp.append(' ' + self.input_name[index] + ' = {' + ','.join(val) + '}') + if len(virt_exp) > 0: + return '\n'.join(virt_exp) + else: + return ' # do ascendc build step' + + def _build_mc2_ctx(self: any): + if len(self.mc2_ctx) != 0: + return '["' + '", "'.join(self.mc2_ctx) + '"]' + return '[]' + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + in_idx = 0 + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + in_idx += 1 + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + now = datetime.datetime.now() + curr_year = now.year + former_year = curr_year - 1 + fd.write(IMPL_HEAD.format(former_year, curr_year, self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): + src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + + def _write_impl(self: any, fd: object, impl_path: str = ""): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + virt_exprs = self._build_virtual() + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, virt_exprs, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file,\ + self.op_type, self.op_file, self.param_type_dynamic, self.op_compile_option)) + else: + if os.environ.get('BUILD_BUILTIN_OPP') == '1': + relative_kernel_src_path = os.path.realpath(self._get_kernel_source(impl_path, src,\ + optype_snake(self.op_type), optype_snake_ex(self.op_type))) + # to match src path in .dat file system, turn relative path into absolute path + abs_rel_kernel_src_path = os.path.join("/", os.path.relpath(relative_kernel_src_path, impl_path)) + + # compiling hidden src file requires src path before packaging .dat file, + # hard code such src path to .py + fd.write(COMPILE_OP_API_BUILT_IN.format(self.op_type, self.op_type,\ + ', '.join(self.input_name), ', '.join(self.output_name), self.param_type_dynamic,\ + self._build_mc2_ctx(), self.input_type + self.output_type, self.output_init_value,\ + self.output_shape_depend_on_compute, self.op_compile_option, abs_rel_kernel_src_path)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.param_type_dynamic, self._build_mc2_ctx(),\ + self.input_type + self.output_type, self.output_init_value, self.output_shape_depend_on_compute,\ + self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + + +class OpFileNotExistsError(Exception): + """File does not exist error.""" + def __str__(self) -> str: + return f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" + + +def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: + """Get all ops info files.""" + ops_info_files = [] + for _dir in opsinfo_dir: + ops_info_files.extend(glob.glob(f'{_dir}/aic-*-ops-info.ini')) + return sorted(ops_info_files) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opsinfo-dir', nargs='*', default=None) + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + + if len(args.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] + + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] + + ops_infos = [] + if args.opsinfo_dir: + ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) + if not ops_infos: + raise OpFileNotExistsError(args.opsinfo_dir) + else: + ops_infos.append(args.argv[1]) + + for ops_info in ops_infos: + write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_ops_config.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..8147f9139 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_ops_config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. +""" + +import os +import glob +import json +import sys +import argparse +import const_var + + +BINARY_INFO_CONFIG_JSON = "binary_info_config.json" + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return sorted(all_suffix_files) + + +def add_dict_key(dict_to_add, key, value): + if value is None: + return + dict_to_add[key] = value + + +def correct_format_mode(format_mode): + if format_mode == 'FormatDefault': + return 'nd_agnostic' + if format_mode == 'FormatAgnostic': + return 'static_nd_agnostic' + if format_mode == 'FormatFixed': + return 'normal' + return format_mode + + +def get_input_or_output_config(in_or_out): + param_dict = {} + name = in_or_out.get('name') + index = in_or_out.get('index') + param_type = in_or_out.get('paramType') + + format_match_mode = in_or_out.get('format_match_mode') + format_mode = correct_format_mode(format_match_mode) + + dtype_mode = in_or_out.get('dtype_match_mode') + if dtype_mode == 'DtypeByte': + dtype_mode = 'bit' + + add_dict_key(param_dict, 'name', name) + add_dict_key(param_dict, 'index', index) + add_dict_key(param_dict, 'paramType', param_type) + add_dict_key(param_dict, 'dtypeMode', dtype_mode) + add_dict_key(param_dict, 'formatMode', format_mode) + return param_dict + + +def get_inputs_or_outputs_config(inputs_or_outputs): + if inputs_or_outputs is None: + return None + inputs_or_outputs_list = [] + + for in_or_out in inputs_or_outputs: + if isinstance(in_or_out, dict): + dict_param_config = get_input_or_output_config(in_or_out) + inputs_or_outputs_list.append(dict_param_config) + elif isinstance(in_or_out, list): + param_info = in_or_out[0] + list_param_config = get_input_or_output_config(param_info) + tmp_list = [list_param_config] + inputs_or_outputs_list.append(tmp_list) + return inputs_or_outputs_list + + +def gen_attrs_config(attrs): + attrs_list = [] + for attr in attrs: + attrs_dict = {} + name = attr.get('name') + mode = attr.get('mode') + add_dict_key(attrs_dict, 'name', name) + add_dict_key(attrs_dict, 'mode', mode) + attrs_list.append(attrs_dict) + return attrs_list + + +def get_params_config(support_info): + params_dict = {} + + inputs = support_info.get('inputs') + inputs_list = get_inputs_or_outputs_config(inputs) + params_dict['inputs'] = inputs_list + + outputs = support_info.get('outputs') + outputs_list = get_inputs_or_outputs_config(outputs) + params_dict['outputs'] = outputs_list + + attrs = support_info.get('attrs') + if attrs is not None: + attrs_list = gen_attrs_config(attrs) + params_dict['attrs'] = attrs_list + + return params_dict + + +def add_simplified_config(op_type, support_info, core_type, task_ration, objfile, config): + simplified_key = support_info.get('simplifiedKey') + + json_path = objfile.split('.')[0] + '.json' + + simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {'dynamicRankSupport': True} + + simplified_key_mode = support_info.get('simplifiedKeyMode') + add_dict_key(op_cfg, 'simplifiedKeyMode', simplified_key_mode) + + optional_input_mode = support_info.get('optionalInputMode') + optional_output_mode = support_info.get('optionalOutputMode') + add_dict_key(op_cfg, 'optionalInputMode', optional_input_mode) + if optional_output_mode is not None: + add_dict_key(op_cfg, 'optionalOutputMode', optional_output_mode) + + params_info = get_params_config(support_info) + op_cfg['params'] = params_info + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'multiKernelType': 1, 'binPath': objfile, 'jsonPath': json_path}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'binPath': objfile, 'jsonPath': json_path}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {'binList': []} + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE': 3, 'MIX_VECTOR_CORE': 4, 'MIX_AIV': 4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = contents.get("coreType") + task_ration = contents.get("taskRation") + core_type = core_type_map.get(core_type, -1) + if core_type == -1 and soc != 'ascend310b': + raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") + + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + + add_dict_key(bin_info, 'implMode', support_info.get('implMode')) + add_dict_key(bin_info, 'int64Mode', support_info.get('int64Mode')) + add_dict_key(bin_info, 'simplifiedKeyMode', support_info.get('simplifiedKeyMode')) + + simplified_key = support_info.get('simplifiedKey') + if simplified_key is not None: + bin_info['simplifiedKey'] = simplified_key + obj_file = os.path.join(soc, op_dir, bin_file_name) + add_simplified_config(op_type, support_info, core_type, task_ration, obj_file, config) + + add_dict_key(bin_info, 'dynamicParamMode', support_info.get('dynamicParamMode')) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + + add_dict_key(bin_info, 'opMode', support_info.get('opMode')) + add_dict_key(bin_info, 'optionalInputMode', support_info.get('optionalInputMode')) + add_dict_key(bin_info, 'deterministic', support_info.get('deterministic')) + if support_info.get('optionalOutputMode') is not None: + add_dict_key(bin_info, 'optionalOutputMode', support_info.get('optionalOutputMode')) + + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def check_single_op_is_void(root_dir): + for root, dirs, _ in os.walk(root_dir): + for sub_dir in dirs: + dir_path = os.path.join(root, sub_dir) + if len(os.listdir(dir_path)) == 0: + print(f"[ERROR] op {sub_dir}: not any obj compile success") + sys.exit(1) + + +def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): + suffix = 'json' + config = {BINARY_INFO_CONFIG_JSON: {}} + check_single_op_is_void(root_dir) + all_json_files = get_specified_suffix_file(root_dir, suffix) + + for _json in all_json_files: + gen_ops_config(_json, soc, config) + file_path = soc + _json.split(soc)[1] + with open(_json, "r+") as f: + data = json.load(f) + data["filePath"] = file_path + f.seek(0) + json.dump(data, f, indent=" ") + f.truncate() + + for cfg_key in config.keys(): + if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: + continue + cfg_file = os.path.join(out_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +# Parse multiple soc_versions ops in single path. +def gen_all_soc_config(all_path): + soc_roots = glob.glob(os.path.join(all_path, "ascend*")) + + for soc_root in soc_roots: + soc = os.path.basename(soc_root) + gen_all_config(soc_root, soc, soc_root, True) + cfg_files = glob.glob(os.path.join(soc_root, "*.json")) + cfg_path = os.path.join(all_path, "config", soc) + os.makedirs(cfg_path, exist_ok=True) + for cfg_file in cfg_files: + new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) + os.rename(cfg_file, new_file) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + + parser.add_argument('-o', + '--out', + nargs='?', + help='Output directory.') + + parser.add_argument('--skip-binary-info-config', + action='store_true', + help='binary_info_config.json file is not parsed.') + + return parser.parse_args() + + +def main(): + args = args_prase() + if args.out is None: + out_dir = args.path + else: + out_dir = args.out + + gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_replay_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/batch_replay_impl.temp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/code_channel_infer.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..3d1e4e87a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product == "ascend910_93" or op_product == "ascend910b": + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/const_var.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/const_var.py new file mode 100755 index 000000000..c7c41d9dd --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/const_var.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', + 'ascend910_93': 'Ascend910_9391', 'ascend610lite': 'Ascend610Lite'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" +SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" +SRC_ENV = ''' +while true; do + case "$1" in + --kernel-src=*) + export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) + shift + ;; + -*) + shift + ;; + *) + break + ;; + esac +done +''' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} + + +def conv_soc_ver(ver: str): + return SOC_MAP_EXT.get(ver) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..b49cf8b9b --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_ops_filter.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_version_info.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..5144edb1c --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/gen_version_info.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_op_info.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_simplified_keys.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..76f8ad286 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/insert_simplified_keys.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_entry.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_impl.temp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/COPYING b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/README.md b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/VERSION b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/make-release.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself-header.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.1 b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.lsm b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/run-tests.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..8576d7616 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/opdesc_parser.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..fa7002b7a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/opdesc_parser.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910b4-1": "ascend910b", + "ascend910_9391": "ascend910_93", + "ascend910_9381": "ascend910_93", + "ascend910_9372": "ascend910_93", + "ascend910_9392": "ascend910_93", + "ascend910_9382": "ascend910_93", + "ascend910_9361": "ascend910_93", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a", + "ascend610lite": "ascend610lite" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_dtype_for_bin_list = [] + self.input_dtype_for_bin = {} + self.input_fmt = [] + self.input_fmt_for_bin_list = [] + self.input_fmt_for_bin = {} + self.input_virt = {} + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_dtype_for_bin_list = [] + self.output_dtype_for_bin = {} + self.output_fmt = [] + self.output_fmt_for_bin_list = [] + self.output_fmt_for_bin = {} + self.output_init_value = [] + self.output_shape_depend_on_compute = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + self.param_type_dynamic = False + self.mc2_ctx = [] + self.bin_cprs_list = [] + self.bin_cprs_head = [] + self.bin_save_list = [] + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + param_type = self._parse_str(conf) + self.input_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_dtype'.format(int(self.input_idx))): + self.input_dtype_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_format'.format(int(self.input_idx))): + self.input_fmt_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.virtual'.format(int(self.input_idx))): + self.input_virt[self.input_idx] = self._parse_str(conf) + elif conf.startswith('input{}.initValue'.format(int(self.input_idx))): + raise Exception(f'[ERROR]: Op: {{\'{self.op_type}\'}} input {self.input_ori_name[int(self.input_idx)]}\ + has InitValue, which is not support!') + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + self.output_init_value.append(None) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + param_type = self._parse_str(conf) + self.output_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_dtype'.format(int(self.output_idx))): + self.output_dtype_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_format'.format(int(self.output_idx))): + self.output_fmt_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.initValue'.format(int(self.output_idx))): + self.output_init_value[int(self.output_idx)] = self._parse_str(conf) + elif conf.startswith('output{}.outputShapeDependOnCompute=true'.format(int(self.output_idx))): + self.output_shape_depend_on_compute.append(int(self.output_idx)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'[ERROR]: The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} \ +conflicts with the built-in variable name. Use a complex name or prefix the operator name.') + + def parse_mc2_ctx(self: any, conf: str): + self.mc2_ctx = self._parse_list(conf) + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options.update(soc_ver_compile_options) + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + break + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('mc2.ctx'): + op_desc.parse_mc2_ctx(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/parse_ini_to_json.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..52dff4ad3 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/parse_ini_to_json.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights£¬group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + parse_ini_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if not ini_file_path_list: + ini_file_path_list.append("tbe_ops_info.ini") + + for ini_file in ini_file_path_list: + if os.path.exists(ini_file): + parse_ini_list.append(ini_file) + + if parse_ini_list: + if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/preset_parse.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/preset_parse.py new file mode 100755 index 000000000..4ff54c0a7 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/preset_parse.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +import json +import sys +import os + + +def read_json(file): + with open(file, 'r') as fd: + config = json.load(fd) + return config + + +def get_config_opts(file): + config = read_json(file) + + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_codegen.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_impl.temp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/tiling_data_def_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..cb2a01c56 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/cmake/util/tiling_data_def_build.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#undef GET_TILING_DATA +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc new file mode 100644 index 000000000..6b6fc00c7 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("Demo") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Demo") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/CMakeLists.txt new file mode 100644 index 000000000..2b8d9034f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/CMakeLists.txt @@ -0,0 +1,93 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +file(GLOB group_proto_src ${ASCEND_AUTOGEN_PATH}/group_proto/*.cc) + +add_library(cust_op_proto SHARED + $<$:${group_proto_src}> + ${ops_srcs} + ${ASCEND_AUTOGEN_PATH}/op_proto.cc +) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +file(GLOB GROUP_PROTO_HEADERS ${ASCEND_AUTOGEN_PATH}/group_proto/*.h) +if (GROUP_PROTO_HEADERS) + install(FILES ${GROUP_PROTO_HEADERS} + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +endif() +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp new file mode 100644 index 000000000..ddcf8818c --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp @@ -0,0 +1,61 @@ + +#include "demo_tiling.h" +#include "register/op_def_registry.h" + + +namespace optiling { +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + + DemoTilingData tiling; + uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); + + tiling.set_size(inputNum); + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class Demo : public OpDef { +public: + explicit Demo(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend910b"); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(Demo); +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h new file mode 100644 index 000000000..9dfd12fa5 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -0,0 +1,10 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(DemoTilingData) + TILING_DATA_FIELD_DEF(uint32_t, size); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp new file mode 100644 index 000000000..937e5211b --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -0,0 +1,57 @@ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue +class KernelDemo { +public: + __aicore__ inline KernelDemo() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, uint32_t totalLength) { + this->totalLength = totalLength; + xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); + yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); + + } + __aicore__ inline void Process() { + CopyIn(); + Compute(); + CopyOut(); + } + +private: + // TODO: user kernel private member + + __aicore__ inline void CopyIn(){ + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::DataCopy(xLocal, xGm, this->totalLength); + inQueueX.EnQue(xLocal); + } + + __aicore__ inline void Compute(){ + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = outQueueY.AllocTensor(); + // sqrt(2) ≈ 1.41 + Muls(yLocal, xLocal, (half)1.41, this->totalLength); + + outQueueY.EnQue(yLocal); + inQueueX.FreeTensor(xLocal); + } + __aicore__ inline void CopyOut(){ + AscendC::LocalTensor yLocal = outQueueY.DeQue(); + AscendC::DataCopy(yGm, yLocal, this->totalLength); + outQueueY.FreeTensor(yLocal); + } +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX; + AscendC::TQue outQueueY; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + uint32_t totalLength; +}; + +extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + KernelDemo op; + op.Init(x, y, tiling_data.size); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/install.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/install.sh new file mode 100755 index 000000000..b87a9790f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + exit 1 + fi +fi +chmod u+w ${targetdir}/vendors + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}\nexport LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + log "[ERROR] echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ export LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH} ] to set the environment path" +fi + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/upgrade.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/upgrade.sh new file mode 100755 index 000000000..65457d2dd --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/scripts/upgrade.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + log "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakeLists.txt new file mode 100644 index 000000000..ea2c509e5 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakeLists.txt @@ -0,0 +1,74 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + if(CMAKE_CROSS_LIBRARY_PATH) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + endif() + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakePresets.json b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakePresets.json new file mode 100644 index 000000000..d32bf6d9a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b;ascend910b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/build.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/build.sh new file mode 100755 index 000000000..113fa60d8 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/build.sh @@ -0,0 +1,95 @@ +#!/bin/bash +if [ -z "$BASE_LIBS_PATH" ]; then + if [ -z "$ASCEND_HOME_PATH" ]; then + if [ -z "$ASCEND_AICPU_PATH" ]; then + echo "please set env." + exit 1 + else + export ASCEND_HOME_PATH=$ASCEND_AICPU_PATH + fi + else + export ASCEND_HOME_PATH=$ASCEND_HOME_PATH + fi +else + export ASCEND_HOME_PATH=$BASE_LIBS_PATH +fi +echo "using ASCEND_HOME_PATH: $ASCEND_HOME_PATH" +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/config.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/config.cmake new file mode 100755 index 000000000..72690b9e4 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/config.cmake @@ -0,0 +1,30 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_AUTOGEN_GROUPPROTO_PATH ${CMAKE_BINARY_DIR}/autogen/group_proto) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH} ${ASCEND_AUTOGEN_GROUPPROTO_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + ) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/func.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/func.cmake new file mode 100755 index 000000000..c4da5fee8 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/func.cmake @@ -0,0 +1,212 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + + set(ENV{ENABLE_SOURCE_PACAKGE} ${OPBUILD_ENABLE_SOURCE}) + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + unset(ENV{ENABLE_SOURCE_PACAKGE}) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_gen_options.py + ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} ${OP_TYPE} ${OP_COMPILE_COMPUTE_UNIT} + ${OP_COMPILE_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR) + if (${EXEC_RESULT}) + message("add ops compile options info: ${EXEC_INFO}") + message("add ops compile options error: ${EXEC_ERROR}") + message(FATAL_ERROR "add ops compile options failed!") + endif() +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + --opc-config-file ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + # add Environment Variable Configurations of python & ccache + set(_ASCENDC_ENV_VAR) + list(APPEND _ASCENDC_ENV_VAR export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} &&) + # whether need judging CMAKE_C_COMPILER_LAUNCHER + if(${CMAKE_CXX_COMPILER_LAUNCHER} MATCHES "ccache$") + list(APPEND _ASCENDC_ENV_VAR export ASCENDC_CCACHE_EXECUTABLE=${CMAKE_CXX_COMPILER_LAUNCHER} &&) + endif() + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND ${_ASCENDC_ENV_VAR} bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/intf.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/intf.cmake new file mode 100755 index 000000000..d2643bbc7 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/intf.cmake @@ -0,0 +1,28 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include + ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel +) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/makeself.cmake b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/makeself.cmake new file mode 100755 index 000000000..2c482b042 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/makeself.cmake @@ -0,0 +1,33 @@ +execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) + +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() + +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + +if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") + execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + + execute_process( + COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) +endif() + + diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/__init__.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_bin_param_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..39bfbb13c --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import argparse +import sys +import os +import json +import hashlib +import re +import copy +from collections import defaultdict +from typing import Dict, List, Set, Tuple, NamedTuple + +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ParamInfo(NamedTuple): + dtype_list: list + format_list: list + dtype_for_bin_list: dict + format_for_bin_list: dict + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + self.tiling_keys = set() + self.op_debug_config = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def set_tiling_key(self: any, tiling_key_info: Set): + if tiling_key_info: + self.tiling_keys.update(tiling_key_info) + + def set_op_debug_config(self: any, op_debug_config: str): + if op_debug_config: + self.op_debug_config = op_debug_config + + def get_full_list(self: any): + dtype_list = [] + for dtype_in in self.input_dtype: + dtype_list.append(dtype_in.split(',')) + for dtype_out in self.output_dtype: + dtype_list.append(dtype_out.split(',')) + + format_list = [] + for fmt_in in self.input_fmt: + format_list.append(fmt_in.split(',')) + for fmt_out in self.output_fmt: + format_list.append(fmt_out.split(',')) + + dtype_for_bin_list = [[] for _ in range(len(self.input_dtype) + len(self.output_dtype))] + format_for_bin_list = copy.deepcopy(dtype_for_bin_list) + + for key, value in self.input_dtype_for_bin.items(): + dtype_for_bin_list[key] = value.split(',') + for key, value in self.output_dtype_for_bin.items(): + dtype_for_bin_list[key + len(self.input_dtype)] = value.split(',') + for key, value in self.input_fmt_for_bin.items(): + format_for_bin_list[key] = value.split(',') + for key, value in self.output_fmt_for_bin.items(): + format_for_bin_list[key + len(self.input_dtype)] = value.split(',') + + return ParamInfo(dtype_list, format_list, dtype_for_bin_list, format_for_bin_list) + + + def gen_bin_cprs_list(self: any, param_info: ParamInfo): + combine_dict = {} + origin_combine_dict = {} + for cob_idx in range(0, len(self.input_dtype[0].split(','))): + origin_combine = "" + combine = "" + for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): + if (param_info.dtype_for_bin_list[param_idx]): + combine += param_info.dtype_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.dtype_list[param_idx][cob_idx] + origin_combine += param_info.dtype_list[param_idx][cob_idx] + if (param_info.format_for_bin_list[param_idx]): + combine += param_info.format_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.format_list[param_idx][cob_idx] + origin_combine += param_info.format_list[param_idx][cob_idx] + if (combine not in combine_dict): + combine_dict[combine] = [] + combine_dict[combine].append(cob_idx) + origin_combine_dict[origin_combine] = cob_idx + for key, value in combine_dict.items(): + if (key not in origin_combine_dict): + print(f"WARNING: ForBinQuery {key} not in origin combine") + self.bin_save_list += value + continue + if len(value) == 1 and value[0] == origin_combine_dict[key]: + self.bin_save_list += value + continue + self.bin_cprs_head.append(origin_combine_dict[key]) + self.bin_cprs_list.append(value) + for index, sub_list in enumerate(self.bin_cprs_list): + if self.bin_cprs_head[index] not in self.bin_save_list: + continue + sub_list.append(self.bin_cprs_head[index]) + self.bin_save_list += self.bin_cprs_head + + + def gen_for_bin_list(self: any, param_info: ParamInfo): + combine_size = len(self.input_dtype[0].split(',')) + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + + self.input_dtype_for_bin_list = [[] for _ in range(input_size)] + self.output_dtype_for_bin_list = [[] for _ in range(output_size)] + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) + self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) + + for index, sub_list in enumerate(self.bin_cprs_list): + head_idx = self.bin_cprs_head[index] + for cmb_idx in sub_list: + for i in range(0, input_size): + self.input_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i][cmb_idx]) + self.input_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i][cmb_idx]) + for i in range(0, output_size): + self.output_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i + input_size][cmb_idx]) + self.output_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i + input_size][cmb_idx]) + + + def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.input_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.input_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.input_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.input_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.input_fmt[i] = new_format_str[:-1] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.output_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.output_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.output_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i + input_size]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.output_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i + input_size]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.output_fmt[i] = new_format_str[:-1] + + + def is_set_for_bin_query(self: any): + return any([ + self.input_dtype_for_bin, + self.output_dtype_for_bin, + self.input_fmt_for_bin, + self.output_fmt_for_bin, + ]) + + + def for_bin_list_match(self: any): + if not self.is_set_for_bin_query(): + return + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + param_info = self.get_full_list() + self.gen_bin_cprs_list(param_info) + self.gen_for_bin_list(param_info) + if len(self.bin_save_list) == len(self.input_dtype[0].split(',')): + print(f'WARNING: ForBinQuery can not compress number of bin file with this set, please check!!.') + return + self.rm_cprs_cmb(param_info.dtype_list, param_info.format_list, input_size, output_size) + + + def gen_input_json(self: any, auto_gen_path: str): + key_map = {} + self.for_bin_list_match() + count = len(self.input_dtype[0].split(',')) + required_parameters = set() + index_value = -1 + + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + required_parameter = [] + op_node = {} + + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + if self.is_set_for_bin_query() and self.input_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.input_dtype_for_bin_list[idx][i] + para['format'] = ifmts[i] + if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.input_fmt_for_bin_list[idx][i] + para['paramType'] = itype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + + input_parameter_key = (idtypes[i], ifmts[i]) + if itype == 'dynamic': + inputs.append([para]) + required_parameter.append(input_parameter_key) + elif itype == 'required': + inputs.append(para) + required_parameter.append(input_parameter_key) + else: + inputs.append(para) + + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + if self.is_set_for_bin_query() and self.output_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.output_dtype_for_bin_list[idx][i] + para['format'] = ofmts[i] + if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.output_fmt_for_bin_list[idx][i] + para['paramType'] = otype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + output_parameter_key = (odtypes[i], ofmts[i]) + if otype == 'dynamic': + outputs.append([para]) + required_parameter.append(output_parameter_key) + elif otype == 'required': + outputs.append(para) + required_parameter.append(output_parameter_key) + else: + outputs.append(para) + + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + + required_parameter_tuple = tuple(required_parameter) + if required_parameter_tuple in required_parameters: + continue + else: + required_parameters.add(required_parameter_tuple) + index_value +=1 + + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) + + + def _write_build_cmd(self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str): + hard_soc = const_var.conv_soc_ver(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + + bin_cmd_str = 'res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ + --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic ' + + build_cmd_var = "#!/bin/bash\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' + plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") + plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") + if plog_level is None: + build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR + if plog_stdout is None: + build_cmd_var += const_var.SET_PLOG_STDOUT + build_cmd_var += const_var.SRC_ENV + if hard_soc == "Ascend610Lite": + build_cmd_var += f'export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n' + build_cmd_var += bin_cmd_str.format(fun=self.op_intf, soc=hard_soc, param=param_file, + impl='high_performance,optional') + enable_tiling_keys = False + if self.tiling_keys: + tiling_keys_list = sorted(list(self.tiling_keys)) + tiling_key_str = ','.join([str(_key) for _key in tiling_keys_list]) + build_cmd_var += f' --tiling_key="{tiling_key_str}"' + enable_tiling_keys = True + + if self.op_debug_config: + op_debug_str = ','.join([str(_key) for _key in list(self.op_debug_config)]) + build_cmd_var += f' --op_debug_config={op_debug_str}' + + build_cmd_var += ")\n" + build_cmd_var += "\n" + if enable_tiling_keys is False: + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + else: + build_cmd_var += "if [ $? -eq 1 ]; then\n" + build_cmd_var += " if echo \"${res}\" | \ +grep -q \"None of the given tiling keys are in the supported list\"; then\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " else\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " exit 1\n" + build_cmd_var += " fi\n" + build_cmd_var += "else\n" + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + build_cmd_var += "fi\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' + + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write(build_cmd_var) + + +def get_tiling_keys(tiling_keys: str) -> Set: + all_tiling_keys = set() + if not tiling_keys: + return all_tiling_keys + + tiling_key_list = tiling_keys.split(';') + for tiling_key_value in tiling_key_list: + pattern = r"(? int(end): + continue + for i in range(int(start), int(end) + 1): + all_tiling_keys.add(i) + elif tiling_key_value.isdigit(): + all_tiling_keys.add(int(tiling_key_value)) + return all_tiling_keys + + +def trans_soc_verion(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: + return low_soc_ver + return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: + tiling_key_info = defaultdict(set) + op_debug_config = defaultdict(set) + if not opc_config_file: + return tiling_key_info, op_debug_config + + if not os.path.exists(opc_config_file): + return tiling_key_info, op_debug_config + + with open(opc_config_file, 'r') as file: + contents = file.readlines() + + for _content in contents: + content = _content.strip() + opc_configs = content.split('@') + if len(opc_configs) < 3: + continue + + op_type = opc_configs[0] + if not op_type: + continue + + compute_unit = opc_configs[1] + if compute_unit: + compute_unit_list = compute_unit.split(';') + soc_lists = [] + for soc_ver in compute_unit_list: + short_soc_ver = trans_soc_verion(soc_ver) + soc_lists.append(short_soc_ver) + if soc not in soc_lists: + continue + + for options in opc_configs[2:]: + if "--tiling_key" in options: + format_tiling_keys = get_tiling_keys(options.split('=')[1]) + if format_tiling_keys: + tiling_key_info[op_type].update(format_tiling_keys) + if "--op_debug_config" in options: + format_debug_config = set(options.split("=")[1].split(';')) + if format_debug_config: + op_debug_config[op_type].update(format_debug_config) + + return tiling_key_info, op_debug_config + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str, + opc_config_file: str = '', ops: list = None): + if not os.path.exists(cfgfile): + print(f'INFO: {cfgfile} does not exists in this project, skip generating compile commands.') + return + + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) + tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) + auto_gen_path_dir = os.path.dirname(cfgfile) + all_soc_key = "ALL" + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + if op_desc.op_type in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) + if all_soc_key in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[all_soc_key]) + if op_desc.op_type in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) + if all_soc_key in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[all_soc_key]) + op_desc.gen_input_json(auto_gen_path_dir) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opc-config-file', nargs='?', const='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + if len(args.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(args.argv[1], + args.argv[2], + args.argv[3], + opc_config_file=args.opc_config_file) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_gen_options.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_gen_options.py new file mode 100755 index 000000000..2dd8ee047 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_gen_options.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import sys +import stat +import os +import re +import json +import const_var + + +def write_options_to_file(file_name: str, options_str: str, \ + op_type: str, compute_unit: str, split_char: str): + flags = os.O_WRONLY | os.O_CREAT + modes = stat.S_IWUSR | stat.S_IRUSR + try: + with os.fdopen(os.open(file_name, flags, modes), 'a') as fd: + fd.write(op_type + split_char + compute_unit + split_char + options_str + '\n') + except Exception as err: + print("write compile options config file failed") + raise(err) + + +def gen_compile_options(compile_options_file: str, op_type: str, \ + compute_unit: str, compile_options: list): + base_dir = os.path.dirname(compile_options_file) + opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") + compile_opt = [] + opc_debug_config = [] + opc_tiling_keys = "" + for opts in compile_options: + if "oom" in opts: + if opts == "--oom": + opc_debug_config.append("oom") + else: + raise RuntimeError(f"Unknown oom option format {opts}") + elif "--save-temp-files" in opts: + opc_debug_config.append("dump_cce") + elif "--tiling_key" in opts: + keys = opts.strip().split('=')[1].split(',') + keys_str = ";".join([key for key in keys]) + opc_tiling_keys = keys_str + else: + compile_opt.append(opts) + if len(compile_opt) > 0: + options_str = ';'.join([opt for opt in compile_opt]) + write_options_to_file(compile_options_file, options_str, op_type, compute_unit, ",") + opc_config_str = "" + if opc_debug_config: + opc_config_str = "--op_debug_config=" + ';'.join([opt for opt in opc_debug_config]) + if len(opc_tiling_keys) > 0: + if opc_config_str != "": + opc_config_str += "@" + opc_config_str += "--tiling_key=" + opc_tiling_keys + + if opc_config_str != "": + write_options_to_file(opc_config_file, opc_config_str, op_type, compute_unit, "@") + + +if __name__ == '__main__': + if len(sys.argv) < 4: + raise RuntimeError('arguments must greater than 4') + compute_soc = "" + comp_options = [] + for i in range(len(sys.argv) - 3): + if sys.argv[i + 3].upper().startswith("ASCEND"): + compute_soc += sys.argv[i + 3] + ";" + else: + comp_options.append(sys.argv[i + 3]) + if compute_soc != "": + compute_soc = compute_soc[0:-1] + gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_impl_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..caa3ab149 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_impl_build.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" + +import argparse +import glob +import sys +import os +import re +import datetime +from typing import List + +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = '''#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. +""" + +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex32": ["DT_COMPLEX32", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) + raise RuntimeError("cannot open %s" %(so_path)) + else: + msg = "load so succ " + so_path + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options.extend(compile_option_list[shortsoc]) + if '__ALLSOC__' in compile_option_list: + compile_options.extend(compile_option_list['__ALLSOC__']) + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): +{} + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = os.environ.get('BISHENG_REAL_PATH') + if bisheng is None: + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_current_build_config("enable_deterministic_mode") == 1: + options.append("-DDETERMINISTIC_MODE=1") + else: + options.append("-DDETERMINISTIC_MODE=0") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + msg = "start replay Acend C Operator {}, kernel name is {}" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API_BUILT_IN = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + + op_compile_option = '{}' + opp_path = os.environ.get('ASCEND_OPP_PATH') + dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) + if opp_path and os.path.exists(dat_path): + # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op + abs_rel_kernel_src_path = "{}" + extend_options = {{}} + extend_options['opp_kernel_hidden_dat_path'] = dat_path + compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ + extend_options) + else: + raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code == 0: + sup = "True" + reason = "" + elif err_code == 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + snake_case += "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + snake_case += "_" + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all: list = None): + self._build_paradefault() + if os.environ.get('BUILD_BUILTIN_OPP') != '1' and impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + print(f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check.") + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd, impl_path) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self: any, op_compile_option_all: list = None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format('OPTION', att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_virtual(self: any) -> str: + virt_exp = [] + for index in range(len(self.input_name)): + if self.input_virt.get(index) is None: + continue + val = [] + val.append('"param_name":"{}"'.format(self.input_name[index])) + val.append('"index":{}'.format(index)) + val.append('"dtype":"{}"'.format(self.input_dtype[index].split(',')[0])) + val.append('"format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"paramType":"optional"') + val.append('"shape":[1]') + val.append('"ori_shape":[1]') + virt_exp.append(' ' + self.input_name[index] + ' = {' + ','.join(val) + '}') + if len(virt_exp) > 0: + return '\n'.join(virt_exp) + else: + return ' # do ascendc build step' + + def _build_mc2_ctx(self: any): + if len(self.mc2_ctx) != 0: + return '["' + '", "'.join(self.mc2_ctx) + '"]' + return '[]' + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + in_idx = 0 + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + in_idx += 1 + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + now = datetime.datetime.now() + curr_year = now.year + former_year = curr_year - 1 + fd.write(IMPL_HEAD.format(former_year, curr_year, self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): + src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + + def _write_impl(self: any, fd: object, impl_path: str = ""): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + virt_exprs = self._build_virtual() + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, virt_exprs, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file,\ + self.op_type, self.op_file, self.param_type_dynamic, self.op_compile_option)) + else: + if os.environ.get('BUILD_BUILTIN_OPP') == '1': + relative_kernel_src_path = os.path.realpath(self._get_kernel_source(impl_path, src,\ + optype_snake(self.op_type), optype_snake_ex(self.op_type))) + # to match src path in .dat file system, turn relative path into absolute path + abs_rel_kernel_src_path = os.path.join("/", os.path.relpath(relative_kernel_src_path, impl_path)) + + # compiling hidden src file requires src path before packaging .dat file, + # hard code such src path to .py + fd.write(COMPILE_OP_API_BUILT_IN.format(self.op_type, self.op_type,\ + ', '.join(self.input_name), ', '.join(self.output_name), self.param_type_dynamic,\ + self._build_mc2_ctx(), self.input_type + self.output_type, self.output_init_value,\ + self.output_shape_depend_on_compute, self.op_compile_option, abs_rel_kernel_src_path)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.param_type_dynamic, self._build_mc2_ctx(),\ + self.input_type + self.output_type, self.output_init_value, self.output_shape_depend_on_compute,\ + self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + + +class OpFileNotExistsError(Exception): + """File does not exist error.""" + def __str__(self) -> str: + return f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" + + +def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: + """Get all ops info files.""" + ops_info_files = [] + for _dir in opsinfo_dir: + ops_info_files.extend(glob.glob(f'{_dir}/aic-*-ops-info.ini')) + return sorted(ops_info_files) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opsinfo-dir', nargs='*', default=None) + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + + if len(args.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] + + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] + + ops_infos = [] + if args.opsinfo_dir: + ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) + if not ops_infos: + raise OpFileNotExistsError(args.opsinfo_dir) + else: + ops_infos.append(args.argv[1]) + + for ops_info in ops_infos: + write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_ops_config.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..8147f9139 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_ops_config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. +""" + +import os +import glob +import json +import sys +import argparse +import const_var + + +BINARY_INFO_CONFIG_JSON = "binary_info_config.json" + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return sorted(all_suffix_files) + + +def add_dict_key(dict_to_add, key, value): + if value is None: + return + dict_to_add[key] = value + + +def correct_format_mode(format_mode): + if format_mode == 'FormatDefault': + return 'nd_agnostic' + if format_mode == 'FormatAgnostic': + return 'static_nd_agnostic' + if format_mode == 'FormatFixed': + return 'normal' + return format_mode + + +def get_input_or_output_config(in_or_out): + param_dict = {} + name = in_or_out.get('name') + index = in_or_out.get('index') + param_type = in_or_out.get('paramType') + + format_match_mode = in_or_out.get('format_match_mode') + format_mode = correct_format_mode(format_match_mode) + + dtype_mode = in_or_out.get('dtype_match_mode') + if dtype_mode == 'DtypeByte': + dtype_mode = 'bit' + + add_dict_key(param_dict, 'name', name) + add_dict_key(param_dict, 'index', index) + add_dict_key(param_dict, 'paramType', param_type) + add_dict_key(param_dict, 'dtypeMode', dtype_mode) + add_dict_key(param_dict, 'formatMode', format_mode) + return param_dict + + +def get_inputs_or_outputs_config(inputs_or_outputs): + if inputs_or_outputs is None: + return None + inputs_or_outputs_list = [] + + for in_or_out in inputs_or_outputs: + if isinstance(in_or_out, dict): + dict_param_config = get_input_or_output_config(in_or_out) + inputs_or_outputs_list.append(dict_param_config) + elif isinstance(in_or_out, list): + param_info = in_or_out[0] + list_param_config = get_input_or_output_config(param_info) + tmp_list = [list_param_config] + inputs_or_outputs_list.append(tmp_list) + return inputs_or_outputs_list + + +def gen_attrs_config(attrs): + attrs_list = [] + for attr in attrs: + attrs_dict = {} + name = attr.get('name') + mode = attr.get('mode') + add_dict_key(attrs_dict, 'name', name) + add_dict_key(attrs_dict, 'mode', mode) + attrs_list.append(attrs_dict) + return attrs_list + + +def get_params_config(support_info): + params_dict = {} + + inputs = support_info.get('inputs') + inputs_list = get_inputs_or_outputs_config(inputs) + params_dict['inputs'] = inputs_list + + outputs = support_info.get('outputs') + outputs_list = get_inputs_or_outputs_config(outputs) + params_dict['outputs'] = outputs_list + + attrs = support_info.get('attrs') + if attrs is not None: + attrs_list = gen_attrs_config(attrs) + params_dict['attrs'] = attrs_list + + return params_dict + + +def add_simplified_config(op_type, support_info, core_type, task_ration, objfile, config): + simplified_key = support_info.get('simplifiedKey') + + json_path = objfile.split('.')[0] + '.json' + + simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {'dynamicRankSupport': True} + + simplified_key_mode = support_info.get('simplifiedKeyMode') + add_dict_key(op_cfg, 'simplifiedKeyMode', simplified_key_mode) + + optional_input_mode = support_info.get('optionalInputMode') + optional_output_mode = support_info.get('optionalOutputMode') + add_dict_key(op_cfg, 'optionalInputMode', optional_input_mode) + if optional_output_mode is not None: + add_dict_key(op_cfg, 'optionalOutputMode', optional_output_mode) + + params_info = get_params_config(support_info) + op_cfg['params'] = params_info + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'multiKernelType': 1, 'binPath': objfile, 'jsonPath': json_path}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'binPath': objfile, 'jsonPath': json_path}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {'binList': []} + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE': 3, 'MIX_VECTOR_CORE': 4, 'MIX_AIV': 4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = contents.get("coreType") + task_ration = contents.get("taskRation") + core_type = core_type_map.get(core_type, -1) + if core_type == -1 and soc != 'ascend310b': + raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") + + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + + add_dict_key(bin_info, 'implMode', support_info.get('implMode')) + add_dict_key(bin_info, 'int64Mode', support_info.get('int64Mode')) + add_dict_key(bin_info, 'simplifiedKeyMode', support_info.get('simplifiedKeyMode')) + + simplified_key = support_info.get('simplifiedKey') + if simplified_key is not None: + bin_info['simplifiedKey'] = simplified_key + obj_file = os.path.join(soc, op_dir, bin_file_name) + add_simplified_config(op_type, support_info, core_type, task_ration, obj_file, config) + + add_dict_key(bin_info, 'dynamicParamMode', support_info.get('dynamicParamMode')) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + + add_dict_key(bin_info, 'opMode', support_info.get('opMode')) + add_dict_key(bin_info, 'optionalInputMode', support_info.get('optionalInputMode')) + add_dict_key(bin_info, 'deterministic', support_info.get('deterministic')) + if support_info.get('optionalOutputMode') is not None: + add_dict_key(bin_info, 'optionalOutputMode', support_info.get('optionalOutputMode')) + + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def check_single_op_is_void(root_dir): + for root, dirs, _ in os.walk(root_dir): + for sub_dir in dirs: + dir_path = os.path.join(root, sub_dir) + if len(os.listdir(dir_path)) == 0: + print(f"[ERROR] op {sub_dir}: not any obj compile success") + sys.exit(1) + + +def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): + suffix = 'json' + config = {BINARY_INFO_CONFIG_JSON: {}} + check_single_op_is_void(root_dir) + all_json_files = get_specified_suffix_file(root_dir, suffix) + + for _json in all_json_files: + gen_ops_config(_json, soc, config) + file_path = soc + _json.split(soc)[1] + with open(_json, "r+") as f: + data = json.load(f) + data["filePath"] = file_path + f.seek(0) + json.dump(data, f, indent=" ") + f.truncate() + + for cfg_key in config.keys(): + if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: + continue + cfg_file = os.path.join(out_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +# Parse multiple soc_versions ops in single path. +def gen_all_soc_config(all_path): + soc_roots = glob.glob(os.path.join(all_path, "ascend*")) + + for soc_root in soc_roots: + soc = os.path.basename(soc_root) + gen_all_config(soc_root, soc, soc_root, True) + cfg_files = glob.glob(os.path.join(soc_root, "*.json")) + cfg_path = os.path.join(all_path, "config", soc) + os.makedirs(cfg_path, exist_ok=True) + for cfg_file in cfg_files: + new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) + os.rename(cfg_file, new_file) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + + parser.add_argument('-o', + '--out', + nargs='?', + help='Output directory.') + + parser.add_argument('--skip-binary-info-config', + action='store_true', + help='binary_info_config.json file is not parsed.') + + return parser.parse_args() + + +def main(): + args = args_prase() + if args.out is None: + out_dir = args.path + else: + out_dir = args.out + + gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_replay_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/batch_replay_impl.temp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/code_channel_infer.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..3d1e4e87a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product == "ascend910_93" or op_product == "ascend910b": + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/const_var.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/const_var.py new file mode 100755 index 000000000..c7c41d9dd --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/const_var.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', + 'ascend910_93': 'Ascend910_9391', 'ascend610lite': 'Ascend610Lite'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" +SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" +SRC_ENV = ''' +while true; do + case "$1" in + --kernel-src=*) + export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) + shift + ;; + -*) + shift + ;; + *) + break + ;; + esac +done +''' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} + + +def conv_soc_ver(ver: str): + return SOC_MAP_EXT.get(ver) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..b49cf8b9b --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_ops_filter.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_version_info.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..5144edb1c --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/gen_version_info.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_op_info.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_simplified_keys.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..76f8ad286 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/insert_simplified_keys.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_entry.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_impl.temp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/COPYING b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/README.md b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/VERSION b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/make-release.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself-header.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.1 b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.lsm b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/run-tests.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..8576d7616 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/opdesc_parser.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..fa7002b7a --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/opdesc_parser.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910b4-1": "ascend910b", + "ascend910_9391": "ascend910_93", + "ascend910_9381": "ascend910_93", + "ascend910_9372": "ascend910_93", + "ascend910_9392": "ascend910_93", + "ascend910_9382": "ascend910_93", + "ascend910_9361": "ascend910_93", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a", + "ascend610lite": "ascend610lite" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_dtype_for_bin_list = [] + self.input_dtype_for_bin = {} + self.input_fmt = [] + self.input_fmt_for_bin_list = [] + self.input_fmt_for_bin = {} + self.input_virt = {} + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_dtype_for_bin_list = [] + self.output_dtype_for_bin = {} + self.output_fmt = [] + self.output_fmt_for_bin_list = [] + self.output_fmt_for_bin = {} + self.output_init_value = [] + self.output_shape_depend_on_compute = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + self.param_type_dynamic = False + self.mc2_ctx = [] + self.bin_cprs_list = [] + self.bin_cprs_head = [] + self.bin_save_list = [] + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + param_type = self._parse_str(conf) + self.input_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_dtype'.format(int(self.input_idx))): + self.input_dtype_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_format'.format(int(self.input_idx))): + self.input_fmt_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.virtual'.format(int(self.input_idx))): + self.input_virt[self.input_idx] = self._parse_str(conf) + elif conf.startswith('input{}.initValue'.format(int(self.input_idx))): + raise Exception(f'[ERROR]: Op: {{\'{self.op_type}\'}} input {self.input_ori_name[int(self.input_idx)]}\ + has InitValue, which is not support!') + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + self.output_init_value.append(None) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + param_type = self._parse_str(conf) + self.output_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_dtype'.format(int(self.output_idx))): + self.output_dtype_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_format'.format(int(self.output_idx))): + self.output_fmt_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.initValue'.format(int(self.output_idx))): + self.output_init_value[int(self.output_idx)] = self._parse_str(conf) + elif conf.startswith('output{}.outputShapeDependOnCompute=true'.format(int(self.output_idx))): + self.output_shape_depend_on_compute.append(int(self.output_idx)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'[ERROR]: The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} \ +conflicts with the built-in variable name. Use a complex name or prefix the operator name.') + + def parse_mc2_ctx(self: any, conf: str): + self.mc2_ctx = self._parse_list(conf) + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options.update(soc_ver_compile_options) + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + break + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('mc2.ctx'): + op_desc.parse_mc2_ctx(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/parse_ini_to_json.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..52dff4ad3 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/parse_ini_to_json.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights£¬group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + parse_ini_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if not ini_file_path_list: + ini_file_path_list.append("tbe_ops_info.ini") + + for ini_file in ini_file_path_list: + if os.path.exists(ini_file): + parse_ini_list.append(ini_file) + + if parse_ini_list: + if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/preset_parse.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/preset_parse.py new file mode 100755 index 000000000..4ff54c0a7 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/preset_parse.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +import json +import sys +import os + + +def read_json(file): + with open(file, 'r') as fd: + config = json.load(fd) + return config + + +def get_config_opts(file): + config = read_json(file) + + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_codegen.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_impl.temp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/tiling_data_def_build.py b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..cb2a01c56 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/cmake/util/tiling_data_def_build.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#undef GET_TILING_DATA +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc new file mode 100644 index 000000000..6b6fc00c7 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("Demo") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Demo") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/CMakeLists.txt new file mode 100644 index 000000000..2b8d9034f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/CMakeLists.txt @@ -0,0 +1,93 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +file(GLOB group_proto_src ${ASCEND_AUTOGEN_PATH}/group_proto/*.cc) + +add_library(cust_op_proto SHARED + $<$:${group_proto_src}> + ${ops_srcs} + ${ASCEND_AUTOGEN_PATH}/op_proto.cc +) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +file(GLOB GROUP_PROTO_HEADERS ${ASCEND_AUTOGEN_PATH}/group_proto/*.h) +if (GROUP_PROTO_HEADERS) + install(FILES ${GROUP_PROTO_HEADERS} + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +endif() +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp new file mode 100644 index 000000000..ddcf8818c --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp @@ -0,0 +1,61 @@ + +#include "demo_tiling.h" +#include "register/op_def_registry.h" + + +namespace optiling { +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + + DemoTilingData tiling; + uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); + + tiling.set_size(inputNum); + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class Demo : public OpDef { +public: + explicit Demo(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend910b"); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(Demo); +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h new file mode 100644 index 000000000..9dfd12fa5 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -0,0 +1,10 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(DemoTilingData) + TILING_DATA_FIELD_DEF(uint32_t, size); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) +} diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/CMakeLists.txt b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp new file mode 100644 index 000000000..b67d910b1 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -0,0 +1,58 @@ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue +class KernelDemo { +public: + __aicore__ inline KernelDemo() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, uint32_t totalLength) { + this->totalLength = totalLength; + xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); + yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); + + } + __aicore__ inline void Process() { + CopyIn(); + Compute(); + CopyOut(); + } + +private: + // TODO: user kernel private member + + __aicore__ inline void CopyIn(){ + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::DataCopy(xLocal, xGm, this->totalLength); + inQueueX.EnQue(xLocal); + } + + __aicore__ inline void Compute(){ + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = outQueueY.AllocTensor(); + half inputVal(2.0); + AscendC::Duplicate(yLocal, inputVal, this->totalLength); + AscendC::Sqrt(yLocal,yLocal,this->totalLength); + Mul(yLocal, xLocal, yLocal, this->totalLength); + outQueueY.EnQue(yLocal); + inQueueX.FreeTensor(xLocal); + } + __aicore__ inline void CopyOut(){ + AscendC::LocalTensor yLocal = outQueueY.DeQue(); + AscendC::DataCopy(yGm, yLocal, this->totalLength); + outQueueY.FreeTensor(yLocal); + } +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX; + AscendC::TQue outQueueY; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + uint32_t totalLength; +}; + +extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + KernelDemo op; + op.Init(x, y, tiling_data.size); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/install.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/install.sh new file mode 100755 index 000000000..b87a9790f --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + exit 1 + fi +fi +chmod u+w ${targetdir}/vendors + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}\nexport LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + log "[ERROR] echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ export LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH} ] to set the environment path" +fi + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/upgrade.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/upgrade.sh new file mode 100755 index 000000000..65457d2dd --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/scripts/upgrade.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + log "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/README.md b/operator_contrib/PrecisionTips/DifferentAlgorithm/README.md new file mode 100644 index 000000000..4a8b4c927 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/README.md @@ -0,0 +1,162 @@ +## 概述 +该样例用于复现强制转换前后顺序导致的精度损失问题。 +## 目录结构介绍 +``` +├── TypeConversion //使用框架调用的方式调用Demo算子 +│ ├── AclNNInvocation // 通过aclnn调用的方式调用Demo算子 +│ ├── Demo // Demo精度异常算子工程 +│ ├── DemoFix // Demo精度正常算子工程 +│ └── Demo.json // Demo算子的原型定义json文件 +``` +## 算子工程介绍 +算子工程目录Demo包含算子实现的模板文件、编译脚本等，如下所示: +``` +├── Demo //Demo自定义算子工程 +│ ├── cmake +│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 +│ ├── op_host // host侧实现文件 +│ ├── op_kernel // kernel侧实现文件 +│ ├── scripts // 自定义算子工程打包相关脚本所在目录 +│ ├── build.sh // 编译入口脚本 +│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt +│ └── CMakePresets.json // 编译配置项 +``` +CANN软件包中提供了工程创建工具msopgen，Demo算子工程可通过Demo.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 +## 编译运行样例算子 +针对自定义算子工程，编译运行包含如下步骤： +- 编译自定义算子工程生成算子安装包； +- 安装自定义算子到算子库中； +- 调用执行自定义算子； + +详细操作如下所示。 +### 1. 获取源码包 +请参考[准备：获取样例代码](../README.md#codeready)完成源码包获取。 +### 2. 修改算子工程配置 + + + - 执行如下命令，切换到算子工程Demo目录。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo + ``` + + - 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + + - 执行如下命令，切换到算子工程DemoFix目录。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix + ``` + + - 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + + +### 3. 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` +### 4. 调用执行算子工程 + +运行run_test.sh，即可复现错误，界面会输出 + ``` +*************** run Demo start**************** +已存在测试数据 +INFO: generate input data success! +可执行存在 +INFO: execute op! +INFO: acl executable run success! +first 10 mistakes data index: 000000, expected: 25.453125000, actual: 25.375000000, rdiff: 0.003069 +first 10 mistakes data index: 000001, expected: 32.750000000, actual: 32.656250000, rdiff: 0.002863 +first 10 mistakes data index: 000002, expected: 27.812500000, actual: 27.734375000, rdiff: 0.002810 +first 10 mistakes data index: 000003, expected: 25.296875000, actual: 25.234375000, rdiff: 0.002470 +first 10 mistakes data index: 000004, expected: 19.968750000, actual: 19.921875000, rdiff: 0.002348 +first 10 mistakes data index: 000005, expected: 29.718750000, actual: 29.640625000, rdiff: 0.002628 +first 10 mistakes data index: 000006, expected: 20.578125000, actual: 20.531250000, rdiff: 0.002277 +first 10 mistakes data index: 000007, expected: 40.468750000, actual: 40.375000000, rdiff: 0.002317 +first 10 mistakes data index: 000008, expected: 43.625000000, actual: 43.531250000, rdiff: 0.002150 +first 10 mistakes data index: 000009, expected: 18.218750000, actual: 18.171875000, rdiff: 0.002573 +first 10 mistakes data index: 000010, expected: 36.093750000, actual: 36.000000000, rdiff: 0.002598 +error ratio: 1.0000, tolrence: 0.0010 +[ERROR] result error +*************** run Demo end**************** +*************** run DemoFix start**************** +已存在测试数据 +INFO: generate input data success! +可执行存在 +INFO: execute op! +INFO: acl executable run success! +error ratio: 0.0000, tolrence: 0.0010 +test pass +*************** run DemoFix end**************** + + +``` + +## 精度调优思路 +修改前代码: +``` + // sqrt(2) ≈ 1.41 + Muls(yLocal, xLocal, (half)1.41, this->totalLength); +``` +sqrt(2) ≈ 1.41，这里的值相对于实际值有精度损失，导致结果精度偏差过大。 + +修改后代码: +``` + half inputVal(2.0); + AscendC::Duplicate(yLocal, inputVal, this->totalLength); + AscendC::Sqrt(yLocal,yLocal,this->totalLength); + Mul(yLocal, xLocal, yLocal, this->totalLength); +``` +修改后sqrt(2)为实际运算的值，精度损失能满足要求，最终结果精度误差也能满足要求。 +这里也可以修改为 + +``` + // sqrt(2) ≈ 1.4142 + Muls(yLocal, xLocal, (half)1.4142, this->totalLength); +``` +由于计算结果偏差要求为小于1e-3,所以sqrt(2)的值设置为1.4142可以满足千分位的精度要求。当sqrt(2)的精度要求更高时，也可以修改为小数位数更多的值，例如:sqrt(2) ≈ 1.4142135623。 \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh new file mode 100755 index 000000000..467a73309 --- /dev/null +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +LOG_PATH=/var/log/npu/slog/debug +if [ -d "$LOG_PATH" ]; then + echo "清理日志，路径：$LOG_PATH" +else + LOG_PATH=~/ascend/log + echo "清理日志，路径：$LOG_PATH" +fi +rm -rf $LOG_PATH* + +current_path=$(pwd) +echo "*************** run Demo start****************" +cd Demo;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 +cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh +echo "*************** run Demo end****************" +cd $current_path +echo "*************** run DemoFix start****************" +cd DemoFix;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 +cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh +echo "*************** run DemoFix end****************" diff --git a/operator_contrib/PrecisionTips/README.md b/operator_contrib/PrecisionTips/README.md new file mode 100644 index 000000000..d1bd352b6 --- /dev/null +++ b/operator_contrib/PrecisionTips/README.md @@ -0,0 +1,7 @@ + +| 目录名称 | 功能描述 | 运行环境 | +| ------------------------------------------------------------ | ---------------------------------------------------- | -- | +| [TypeConversion](./TypeConversion) | 类型转换顺序导致误差样例 | Atlas A2训练产品 | +| [DifferentAlgorithm](./DifferentAlgorithm) | 算法导致精度误差样例 | Atlas A2训练产品| +| [SequenceOperationSteps](./SequenceOperationSteps) | 运算步骤顺序导致精度误差样例 | Atlas A2训练产品| + diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/common.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/op_runner.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/operator_desc.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..17607353d --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,60 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + bool keepDims; + bool ignoreNan; + char *dtype; + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/run.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/run.sh new file mode 100644 index 000000000..98c688714 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/run.sh @@ -0,0 +1,109 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + # rm ./input/*.bin + rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_op >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output.bin output/golden.bin + +} + +main diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/acl.json b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/gen_data.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..979604de4 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,28 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os +np.random.seed(0) +def gen_golden_data_simple(): + input_x = np.random.uniform(0, 65503, [1024]).astype(np.float16) + input_y = np.random.uniform(0, 65503, [1024]).astype(np.float16) + golden = input_x * 0.001 + input_y * 0.001 + + print(golden) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + input_y.tofile("./input/input_y.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/verify_result.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..3b5751e1f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "first 10 mistakes data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 10: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..ee73e930f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include + ${BUILT_PKG_PATH}/include/aclnnop +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/main.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..7db4582a2 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/main.cpp @@ -0,0 +1,165 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{1024}; + std::vector shape_out{1024}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape_out.size(), shape_out.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_y.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..61911842a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,485 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_demo.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto ret = aclDestroyTensor(inputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(inputBuffers_[i]); + + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devInputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostInputs_[i]); + } else { + ret = aclrtFreeHost(hostInputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto ret = aclDestroyTensor(outputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(outputBuffers_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devOutputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostOutputs_[i]); + } else { + ret = aclrtFreeHost(hostOutputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + + auto ret = aclnnDemoGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + ret = aclnnDemo(workspace, workspaceSize, handle, stream); + + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute Operator success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo.json b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo.json new file mode 100644 index 000000000..05e6dbb25 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo.json @@ -0,0 +1,28 @@ +[ + { + "op": "Demo", + "language": "cpp", + "input_desc": [ + { + "name": "x", + "param_type": "required", + "format": ["ND"], + "type": ["float16"] + }, + { + "name": "y", + "param_type": "required", + "format": ["ND"], + "type": ["float16"] + } + ], + "output_desc": [ + { + "name": "z", + "param_type": "required", + "format": ["ND"], + "type": ["float16"] + } + ] + } +] \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakeLists.txt new file mode 100644 index 000000000..ea2c509e5 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakeLists.txt @@ -0,0 +1,74 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + if(CMAKE_CROSS_LIBRARY_PATH) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + endif() + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakePresets.json b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakePresets.json new file mode 100644 index 000000000..d32bf6d9a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b;ascend910b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/build.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/build.sh new file mode 100755 index 000000000..113fa60d8 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/build.sh @@ -0,0 +1,95 @@ +#!/bin/bash +if [ -z "$BASE_LIBS_PATH" ]; then + if [ -z "$ASCEND_HOME_PATH" ]; then + if [ -z "$ASCEND_AICPU_PATH" ]; then + echo "please set env." + exit 1 + else + export ASCEND_HOME_PATH=$ASCEND_AICPU_PATH + fi + else + export ASCEND_HOME_PATH=$ASCEND_HOME_PATH + fi +else + export ASCEND_HOME_PATH=$BASE_LIBS_PATH +fi +echo "using ASCEND_HOME_PATH: $ASCEND_HOME_PATH" +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/config.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/config.cmake new file mode 100755 index 000000000..72690b9e4 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/config.cmake @@ -0,0 +1,30 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_AUTOGEN_GROUPPROTO_PATH ${CMAKE_BINARY_DIR}/autogen/group_proto) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH} ${ASCEND_AUTOGEN_GROUPPROTO_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + ) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/func.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/func.cmake new file mode 100755 index 000000000..c4da5fee8 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/func.cmake @@ -0,0 +1,212 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + + set(ENV{ENABLE_SOURCE_PACAKGE} ${OPBUILD_ENABLE_SOURCE}) + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + unset(ENV{ENABLE_SOURCE_PACAKGE}) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_gen_options.py + ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} ${OP_TYPE} ${OP_COMPILE_COMPUTE_UNIT} + ${OP_COMPILE_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR) + if (${EXEC_RESULT}) + message("add ops compile options info: ${EXEC_INFO}") + message("add ops compile options error: ${EXEC_ERROR}") + message(FATAL_ERROR "add ops compile options failed!") + endif() +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + --opc-config-file ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + # add Environment Variable Configurations of python & ccache + set(_ASCENDC_ENV_VAR) + list(APPEND _ASCENDC_ENV_VAR export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} &&) + # whether need judging CMAKE_C_COMPILER_LAUNCHER + if(${CMAKE_CXX_COMPILER_LAUNCHER} MATCHES "ccache$") + list(APPEND _ASCENDC_ENV_VAR export ASCENDC_CCACHE_EXECUTABLE=${CMAKE_CXX_COMPILER_LAUNCHER} &&) + endif() + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND ${_ASCENDC_ENV_VAR} bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/intf.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/intf.cmake new file mode 100755 index 000000000..d2643bbc7 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/intf.cmake @@ -0,0 +1,28 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include + ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel +) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/makeself.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/makeself.cmake new file mode 100755 index 000000000..2c482b042 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/makeself.cmake @@ -0,0 +1,33 @@ +execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) + +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() + +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + +if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") + execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + + execute_process( + COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) +endif() + + diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/__init__.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_bin_param_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..39bfbb13c --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import argparse +import sys +import os +import json +import hashlib +import re +import copy +from collections import defaultdict +from typing import Dict, List, Set, Tuple, NamedTuple + +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ParamInfo(NamedTuple): + dtype_list: list + format_list: list + dtype_for_bin_list: dict + format_for_bin_list: dict + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + self.tiling_keys = set() + self.op_debug_config = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def set_tiling_key(self: any, tiling_key_info: Set): + if tiling_key_info: + self.tiling_keys.update(tiling_key_info) + + def set_op_debug_config(self: any, op_debug_config: str): + if op_debug_config: + self.op_debug_config = op_debug_config + + def get_full_list(self: any): + dtype_list = [] + for dtype_in in self.input_dtype: + dtype_list.append(dtype_in.split(',')) + for dtype_out in self.output_dtype: + dtype_list.append(dtype_out.split(',')) + + format_list = [] + for fmt_in in self.input_fmt: + format_list.append(fmt_in.split(',')) + for fmt_out in self.output_fmt: + format_list.append(fmt_out.split(',')) + + dtype_for_bin_list = [[] for _ in range(len(self.input_dtype) + len(self.output_dtype))] + format_for_bin_list = copy.deepcopy(dtype_for_bin_list) + + for key, value in self.input_dtype_for_bin.items(): + dtype_for_bin_list[key] = value.split(',') + for key, value in self.output_dtype_for_bin.items(): + dtype_for_bin_list[key + len(self.input_dtype)] = value.split(',') + for key, value in self.input_fmt_for_bin.items(): + format_for_bin_list[key] = value.split(',') + for key, value in self.output_fmt_for_bin.items(): + format_for_bin_list[key + len(self.input_dtype)] = value.split(',') + + return ParamInfo(dtype_list, format_list, dtype_for_bin_list, format_for_bin_list) + + + def gen_bin_cprs_list(self: any, param_info: ParamInfo): + combine_dict = {} + origin_combine_dict = {} + for cob_idx in range(0, len(self.input_dtype[0].split(','))): + origin_combine = "" + combine = "" + for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): + if (param_info.dtype_for_bin_list[param_idx]): + combine += param_info.dtype_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.dtype_list[param_idx][cob_idx] + origin_combine += param_info.dtype_list[param_idx][cob_idx] + if (param_info.format_for_bin_list[param_idx]): + combine += param_info.format_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.format_list[param_idx][cob_idx] + origin_combine += param_info.format_list[param_idx][cob_idx] + if (combine not in combine_dict): + combine_dict[combine] = [] + combine_dict[combine].append(cob_idx) + origin_combine_dict[origin_combine] = cob_idx + for key, value in combine_dict.items(): + if (key not in origin_combine_dict): + print(f"WARNING: ForBinQuery {key} not in origin combine") + self.bin_save_list += value + continue + if len(value) == 1 and value[0] == origin_combine_dict[key]: + self.bin_save_list += value + continue + self.bin_cprs_head.append(origin_combine_dict[key]) + self.bin_cprs_list.append(value) + for index, sub_list in enumerate(self.bin_cprs_list): + if self.bin_cprs_head[index] not in self.bin_save_list: + continue + sub_list.append(self.bin_cprs_head[index]) + self.bin_save_list += self.bin_cprs_head + + + def gen_for_bin_list(self: any, param_info: ParamInfo): + combine_size = len(self.input_dtype[0].split(',')) + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + + self.input_dtype_for_bin_list = [[] for _ in range(input_size)] + self.output_dtype_for_bin_list = [[] for _ in range(output_size)] + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) + self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) + + for index, sub_list in enumerate(self.bin_cprs_list): + head_idx = self.bin_cprs_head[index] + for cmb_idx in sub_list: + for i in range(0, input_size): + self.input_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i][cmb_idx]) + self.input_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i][cmb_idx]) + for i in range(0, output_size): + self.output_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i + input_size][cmb_idx]) + self.output_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i + input_size][cmb_idx]) + + + def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.input_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.input_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.input_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.input_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.input_fmt[i] = new_format_str[:-1] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.output_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.output_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.output_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i + input_size]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.output_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i + input_size]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.output_fmt[i] = new_format_str[:-1] + + + def is_set_for_bin_query(self: any): + return any([ + self.input_dtype_for_bin, + self.output_dtype_for_bin, + self.input_fmt_for_bin, + self.output_fmt_for_bin, + ]) + + + def for_bin_list_match(self: any): + if not self.is_set_for_bin_query(): + return + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + param_info = self.get_full_list() + self.gen_bin_cprs_list(param_info) + self.gen_for_bin_list(param_info) + if len(self.bin_save_list) == len(self.input_dtype[0].split(',')): + print(f'WARNING: ForBinQuery can not compress number of bin file with this set, please check!!.') + return + self.rm_cprs_cmb(param_info.dtype_list, param_info.format_list, input_size, output_size) + + + def gen_input_json(self: any, auto_gen_path: str): + key_map = {} + self.for_bin_list_match() + count = len(self.input_dtype[0].split(',')) + required_parameters = set() + index_value = -1 + + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + required_parameter = [] + op_node = {} + + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + if self.is_set_for_bin_query() and self.input_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.input_dtype_for_bin_list[idx][i] + para['format'] = ifmts[i] + if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.input_fmt_for_bin_list[idx][i] + para['paramType'] = itype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + + input_parameter_key = (idtypes[i], ifmts[i]) + if itype == 'dynamic': + inputs.append([para]) + required_parameter.append(input_parameter_key) + elif itype == 'required': + inputs.append(para) + required_parameter.append(input_parameter_key) + else: + inputs.append(para) + + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + if self.is_set_for_bin_query() and self.output_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.output_dtype_for_bin_list[idx][i] + para['format'] = ofmts[i] + if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.output_fmt_for_bin_list[idx][i] + para['paramType'] = otype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + output_parameter_key = (odtypes[i], ofmts[i]) + if otype == 'dynamic': + outputs.append([para]) + required_parameter.append(output_parameter_key) + elif otype == 'required': + outputs.append(para) + required_parameter.append(output_parameter_key) + else: + outputs.append(para) + + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + + required_parameter_tuple = tuple(required_parameter) + if required_parameter_tuple in required_parameters: + continue + else: + required_parameters.add(required_parameter_tuple) + index_value +=1 + + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) + + + def _write_build_cmd(self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str): + hard_soc = const_var.conv_soc_ver(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + + bin_cmd_str = 'res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ + --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic ' + + build_cmd_var = "#!/bin/bash\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' + plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") + plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") + if plog_level is None: + build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR + if plog_stdout is None: + build_cmd_var += const_var.SET_PLOG_STDOUT + build_cmd_var += const_var.SRC_ENV + if hard_soc == "Ascend610Lite": + build_cmd_var += f'export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n' + build_cmd_var += bin_cmd_str.format(fun=self.op_intf, soc=hard_soc, param=param_file, + impl='high_performance,optional') + enable_tiling_keys = False + if self.tiling_keys: + tiling_keys_list = sorted(list(self.tiling_keys)) + tiling_key_str = ','.join([str(_key) for _key in tiling_keys_list]) + build_cmd_var += f' --tiling_key="{tiling_key_str}"' + enable_tiling_keys = True + + if self.op_debug_config: + op_debug_str = ','.join([str(_key) for _key in list(self.op_debug_config)]) + build_cmd_var += f' --op_debug_config={op_debug_str}' + + build_cmd_var += ")\n" + build_cmd_var += "\n" + if enable_tiling_keys is False: + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + else: + build_cmd_var += "if [ $? -eq 1 ]; then\n" + build_cmd_var += " if echo \"${res}\" | \ +grep -q \"None of the given tiling keys are in the supported list\"; then\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " else\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " exit 1\n" + build_cmd_var += " fi\n" + build_cmd_var += "else\n" + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + build_cmd_var += "fi\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' + + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write(build_cmd_var) + + +def get_tiling_keys(tiling_keys: str) -> Set: + all_tiling_keys = set() + if not tiling_keys: + return all_tiling_keys + + tiling_key_list = tiling_keys.split(';') + for tiling_key_value in tiling_key_list: + pattern = r"(? int(end): + continue + for i in range(int(start), int(end) + 1): + all_tiling_keys.add(i) + elif tiling_key_value.isdigit(): + all_tiling_keys.add(int(tiling_key_value)) + return all_tiling_keys + + +def trans_soc_verion(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: + return low_soc_ver + return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: + tiling_key_info = defaultdict(set) + op_debug_config = defaultdict(set) + if not opc_config_file: + return tiling_key_info, op_debug_config + + if not os.path.exists(opc_config_file): + return tiling_key_info, op_debug_config + + with open(opc_config_file, 'r') as file: + contents = file.readlines() + + for _content in contents: + content = _content.strip() + opc_configs = content.split('@') + if len(opc_configs) < 3: + continue + + op_type = opc_configs[0] + if not op_type: + continue + + compute_unit = opc_configs[1] + if compute_unit: + compute_unit_list = compute_unit.split(';') + soc_lists = [] + for soc_ver in compute_unit_list: + short_soc_ver = trans_soc_verion(soc_ver) + soc_lists.append(short_soc_ver) + if soc not in soc_lists: + continue + + for options in opc_configs[2:]: + if "--tiling_key" in options: + format_tiling_keys = get_tiling_keys(options.split('=')[1]) + if format_tiling_keys: + tiling_key_info[op_type].update(format_tiling_keys) + if "--op_debug_config" in options: + format_debug_config = set(options.split("=")[1].split(';')) + if format_debug_config: + op_debug_config[op_type].update(format_debug_config) + + return tiling_key_info, op_debug_config + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str, + opc_config_file: str = '', ops: list = None): + if not os.path.exists(cfgfile): + print(f'INFO: {cfgfile} does not exists in this project, skip generating compile commands.') + return + + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) + tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) + auto_gen_path_dir = os.path.dirname(cfgfile) + all_soc_key = "ALL" + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + if op_desc.op_type in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) + if all_soc_key in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[all_soc_key]) + if op_desc.op_type in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) + if all_soc_key in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[all_soc_key]) + op_desc.gen_input_json(auto_gen_path_dir) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opc-config-file', nargs='?', const='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + if len(args.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(args.argv[1], + args.argv[2], + args.argv[3], + opc_config_file=args.opc_config_file) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_gen_options.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_gen_options.py new file mode 100755 index 000000000..2dd8ee047 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_gen_options.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import sys +import stat +import os +import re +import json +import const_var + + +def write_options_to_file(file_name: str, options_str: str, \ + op_type: str, compute_unit: str, split_char: str): + flags = os.O_WRONLY | os.O_CREAT + modes = stat.S_IWUSR | stat.S_IRUSR + try: + with os.fdopen(os.open(file_name, flags, modes), 'a') as fd: + fd.write(op_type + split_char + compute_unit + split_char + options_str + '\n') + except Exception as err: + print("write compile options config file failed") + raise(err) + + +def gen_compile_options(compile_options_file: str, op_type: str, \ + compute_unit: str, compile_options: list): + base_dir = os.path.dirname(compile_options_file) + opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") + compile_opt = [] + opc_debug_config = [] + opc_tiling_keys = "" + for opts in compile_options: + if "oom" in opts: + if opts == "--oom": + opc_debug_config.append("oom") + else: + raise RuntimeError(f"Unknown oom option format {opts}") + elif "--save-temp-files" in opts: + opc_debug_config.append("dump_cce") + elif "--tiling_key" in opts: + keys = opts.strip().split('=')[1].split(',') + keys_str = ";".join([key for key in keys]) + opc_tiling_keys = keys_str + else: + compile_opt.append(opts) + if len(compile_opt) > 0: + options_str = ';'.join([opt for opt in compile_opt]) + write_options_to_file(compile_options_file, options_str, op_type, compute_unit, ",") + opc_config_str = "" + if opc_debug_config: + opc_config_str = "--op_debug_config=" + ';'.join([opt for opt in opc_debug_config]) + if len(opc_tiling_keys) > 0: + if opc_config_str != "": + opc_config_str += "@" + opc_config_str += "--tiling_key=" + opc_tiling_keys + + if opc_config_str != "": + write_options_to_file(opc_config_file, opc_config_str, op_type, compute_unit, "@") + + +if __name__ == '__main__': + if len(sys.argv) < 4: + raise RuntimeError('arguments must greater than 4') + compute_soc = "" + comp_options = [] + for i in range(len(sys.argv) - 3): + if sys.argv[i + 3].upper().startswith("ASCEND"): + compute_soc += sys.argv[i + 3] + ";" + else: + comp_options.append(sys.argv[i + 3]) + if compute_soc != "": + compute_soc = compute_soc[0:-1] + gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_impl_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..caa3ab149 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_impl_build.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" + +import argparse +import glob +import sys +import os +import re +import datetime +from typing import List + +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = '''#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. +""" + +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex32": ["DT_COMPLEX32", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) + raise RuntimeError("cannot open %s" %(so_path)) + else: + msg = "load so succ " + so_path + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options.extend(compile_option_list[shortsoc]) + if '__ALLSOC__' in compile_option_list: + compile_options.extend(compile_option_list['__ALLSOC__']) + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): +{} + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = os.environ.get('BISHENG_REAL_PATH') + if bisheng is None: + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_current_build_config("enable_deterministic_mode") == 1: + options.append("-DDETERMINISTIC_MODE=1") + else: + options.append("-DDETERMINISTIC_MODE=0") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + msg = "start replay Acend C Operator {}, kernel name is {}" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API_BUILT_IN = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + + op_compile_option = '{}' + opp_path = os.environ.get('ASCEND_OPP_PATH') + dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) + if opp_path and os.path.exists(dat_path): + # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op + abs_rel_kernel_src_path = "{}" + extend_options = {{}} + extend_options['opp_kernel_hidden_dat_path'] = dat_path + compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ + extend_options) + else: + raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code == 0: + sup = "True" + reason = "" + elif err_code == 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + snake_case += "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + snake_case += "_" + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all: list = None): + self._build_paradefault() + if os.environ.get('BUILD_BUILTIN_OPP') != '1' and impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + print(f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check.") + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd, impl_path) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self: any, op_compile_option_all: list = None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format('OPTION', att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_virtual(self: any) -> str: + virt_exp = [] + for index in range(len(self.input_name)): + if self.input_virt.get(index) is None: + continue + val = [] + val.append('"param_name":"{}"'.format(self.input_name[index])) + val.append('"index":{}'.format(index)) + val.append('"dtype":"{}"'.format(self.input_dtype[index].split(',')[0])) + val.append('"format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"paramType":"optional"') + val.append('"shape":[1]') + val.append('"ori_shape":[1]') + virt_exp.append(' ' + self.input_name[index] + ' = {' + ','.join(val) + '}') + if len(virt_exp) > 0: + return '\n'.join(virt_exp) + else: + return ' # do ascendc build step' + + def _build_mc2_ctx(self: any): + if len(self.mc2_ctx) != 0: + return '["' + '", "'.join(self.mc2_ctx) + '"]' + return '[]' + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + in_idx = 0 + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + in_idx += 1 + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + now = datetime.datetime.now() + curr_year = now.year + former_year = curr_year - 1 + fd.write(IMPL_HEAD.format(former_year, curr_year, self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): + src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + + def _write_impl(self: any, fd: object, impl_path: str = ""): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + virt_exprs = self._build_virtual() + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, virt_exprs, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file,\ + self.op_type, self.op_file, self.param_type_dynamic, self.op_compile_option)) + else: + if os.environ.get('BUILD_BUILTIN_OPP') == '1': + relative_kernel_src_path = os.path.realpath(self._get_kernel_source(impl_path, src,\ + optype_snake(self.op_type), optype_snake_ex(self.op_type))) + # to match src path in .dat file system, turn relative path into absolute path + abs_rel_kernel_src_path = os.path.join("/", os.path.relpath(relative_kernel_src_path, impl_path)) + + # compiling hidden src file requires src path before packaging .dat file, + # hard code such src path to .py + fd.write(COMPILE_OP_API_BUILT_IN.format(self.op_type, self.op_type,\ + ', '.join(self.input_name), ', '.join(self.output_name), self.param_type_dynamic,\ + self._build_mc2_ctx(), self.input_type + self.output_type, self.output_init_value,\ + self.output_shape_depend_on_compute, self.op_compile_option, abs_rel_kernel_src_path)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.param_type_dynamic, self._build_mc2_ctx(),\ + self.input_type + self.output_type, self.output_init_value, self.output_shape_depend_on_compute,\ + self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + + +class OpFileNotExistsError(Exception): + """File does not exist error.""" + def __str__(self) -> str: + return f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" + + +def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: + """Get all ops info files.""" + ops_info_files = [] + for _dir in opsinfo_dir: + ops_info_files.extend(glob.glob(f'{_dir}/aic-*-ops-info.ini')) + return sorted(ops_info_files) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opsinfo-dir', nargs='*', default=None) + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + + if len(args.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] + + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] + + ops_infos = [] + if args.opsinfo_dir: + ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) + if not ops_infos: + raise OpFileNotExistsError(args.opsinfo_dir) + else: + ops_infos.append(args.argv[1]) + + for ops_info in ops_infos: + write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_ops_config.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..8147f9139 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_ops_config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. +""" + +import os +import glob +import json +import sys +import argparse +import const_var + + +BINARY_INFO_CONFIG_JSON = "binary_info_config.json" + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return sorted(all_suffix_files) + + +def add_dict_key(dict_to_add, key, value): + if value is None: + return + dict_to_add[key] = value + + +def correct_format_mode(format_mode): + if format_mode == 'FormatDefault': + return 'nd_agnostic' + if format_mode == 'FormatAgnostic': + return 'static_nd_agnostic' + if format_mode == 'FormatFixed': + return 'normal' + return format_mode + + +def get_input_or_output_config(in_or_out): + param_dict = {} + name = in_or_out.get('name') + index = in_or_out.get('index') + param_type = in_or_out.get('paramType') + + format_match_mode = in_or_out.get('format_match_mode') + format_mode = correct_format_mode(format_match_mode) + + dtype_mode = in_or_out.get('dtype_match_mode') + if dtype_mode == 'DtypeByte': + dtype_mode = 'bit' + + add_dict_key(param_dict, 'name', name) + add_dict_key(param_dict, 'index', index) + add_dict_key(param_dict, 'paramType', param_type) + add_dict_key(param_dict, 'dtypeMode', dtype_mode) + add_dict_key(param_dict, 'formatMode', format_mode) + return param_dict + + +def get_inputs_or_outputs_config(inputs_or_outputs): + if inputs_or_outputs is None: + return None + inputs_or_outputs_list = [] + + for in_or_out in inputs_or_outputs: + if isinstance(in_or_out, dict): + dict_param_config = get_input_or_output_config(in_or_out) + inputs_or_outputs_list.append(dict_param_config) + elif isinstance(in_or_out, list): + param_info = in_or_out[0] + list_param_config = get_input_or_output_config(param_info) + tmp_list = [list_param_config] + inputs_or_outputs_list.append(tmp_list) + return inputs_or_outputs_list + + +def gen_attrs_config(attrs): + attrs_list = [] + for attr in attrs: + attrs_dict = {} + name = attr.get('name') + mode = attr.get('mode') + add_dict_key(attrs_dict, 'name', name) + add_dict_key(attrs_dict, 'mode', mode) + attrs_list.append(attrs_dict) + return attrs_list + + +def get_params_config(support_info): + params_dict = {} + + inputs = support_info.get('inputs') + inputs_list = get_inputs_or_outputs_config(inputs) + params_dict['inputs'] = inputs_list + + outputs = support_info.get('outputs') + outputs_list = get_inputs_or_outputs_config(outputs) + params_dict['outputs'] = outputs_list + + attrs = support_info.get('attrs') + if attrs is not None: + attrs_list = gen_attrs_config(attrs) + params_dict['attrs'] = attrs_list + + return params_dict + + +def add_simplified_config(op_type, support_info, core_type, task_ration, objfile, config): + simplified_key = support_info.get('simplifiedKey') + + json_path = objfile.split('.')[0] + '.json' + + simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {'dynamicRankSupport': True} + + simplified_key_mode = support_info.get('simplifiedKeyMode') + add_dict_key(op_cfg, 'simplifiedKeyMode', simplified_key_mode) + + optional_input_mode = support_info.get('optionalInputMode') + optional_output_mode = support_info.get('optionalOutputMode') + add_dict_key(op_cfg, 'optionalInputMode', optional_input_mode) + if optional_output_mode is not None: + add_dict_key(op_cfg, 'optionalOutputMode', optional_output_mode) + + params_info = get_params_config(support_info) + op_cfg['params'] = params_info + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'multiKernelType': 1, 'binPath': objfile, 'jsonPath': json_path}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'binPath': objfile, 'jsonPath': json_path}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {'binList': []} + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE': 3, 'MIX_VECTOR_CORE': 4, 'MIX_AIV': 4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = contents.get("coreType") + task_ration = contents.get("taskRation") + core_type = core_type_map.get(core_type, -1) + if core_type == -1 and soc != 'ascend310b': + raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") + + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + + add_dict_key(bin_info, 'implMode', support_info.get('implMode')) + add_dict_key(bin_info, 'int64Mode', support_info.get('int64Mode')) + add_dict_key(bin_info, 'simplifiedKeyMode', support_info.get('simplifiedKeyMode')) + + simplified_key = support_info.get('simplifiedKey') + if simplified_key is not None: + bin_info['simplifiedKey'] = simplified_key + obj_file = os.path.join(soc, op_dir, bin_file_name) + add_simplified_config(op_type, support_info, core_type, task_ration, obj_file, config) + + add_dict_key(bin_info, 'dynamicParamMode', support_info.get('dynamicParamMode')) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + + add_dict_key(bin_info, 'opMode', support_info.get('opMode')) + add_dict_key(bin_info, 'optionalInputMode', support_info.get('optionalInputMode')) + add_dict_key(bin_info, 'deterministic', support_info.get('deterministic')) + if support_info.get('optionalOutputMode') is not None: + add_dict_key(bin_info, 'optionalOutputMode', support_info.get('optionalOutputMode')) + + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def check_single_op_is_void(root_dir): + for root, dirs, _ in os.walk(root_dir): + for sub_dir in dirs: + dir_path = os.path.join(root, sub_dir) + if len(os.listdir(dir_path)) == 0: + print(f"[ERROR] op {sub_dir}: not any obj compile success") + sys.exit(1) + + +def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): + suffix = 'json' + config = {BINARY_INFO_CONFIG_JSON: {}} + check_single_op_is_void(root_dir) + all_json_files = get_specified_suffix_file(root_dir, suffix) + + for _json in all_json_files: + gen_ops_config(_json, soc, config) + file_path = soc + _json.split(soc)[1] + with open(_json, "r+") as f: + data = json.load(f) + data["filePath"] = file_path + f.seek(0) + json.dump(data, f, indent=" ") + f.truncate() + + for cfg_key in config.keys(): + if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: + continue + cfg_file = os.path.join(out_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +# Parse multiple soc_versions ops in single path. +def gen_all_soc_config(all_path): + soc_roots = glob.glob(os.path.join(all_path, "ascend*")) + + for soc_root in soc_roots: + soc = os.path.basename(soc_root) + gen_all_config(soc_root, soc, soc_root, True) + cfg_files = glob.glob(os.path.join(soc_root, "*.json")) + cfg_path = os.path.join(all_path, "config", soc) + os.makedirs(cfg_path, exist_ok=True) + for cfg_file in cfg_files: + new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) + os.rename(cfg_file, new_file) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + + parser.add_argument('-o', + '--out', + nargs='?', + help='Output directory.') + + parser.add_argument('--skip-binary-info-config', + action='store_true', + help='binary_info_config.json file is not parsed.') + + return parser.parse_args() + + +def main(): + args = args_prase() + if args.out is None: + out_dir = args.path + else: + out_dir = args.out + + gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_replay_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/batch_replay_impl.temp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/code_channel_infer.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..3d1e4e87a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product == "ascend910_93" or op_product == "ascend910b": + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/const_var.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/const_var.py new file mode 100755 index 000000000..c7c41d9dd --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/const_var.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', + 'ascend910_93': 'Ascend910_9391', 'ascend610lite': 'Ascend610Lite'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" +SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" +SRC_ENV = ''' +while true; do + case "$1" in + --kernel-src=*) + export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) + shift + ;; + -*) + shift + ;; + *) + break + ;; + esac +done +''' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} + + +def conv_soc_ver(ver: str): + return SOC_MAP_EXT.get(ver) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..b49cf8b9b --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_ops_filter.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_version_info.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..5144edb1c --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/gen_version_info.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_op_info.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_simplified_keys.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..76f8ad286 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/insert_simplified_keys.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_entry.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_impl.temp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/COPYING b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/README.md b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/VERSION b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/make-release.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself-header.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.1 b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.lsm b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/run-tests.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..8576d7616 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/opdesc_parser.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..fa7002b7a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/opdesc_parser.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910b4-1": "ascend910b", + "ascend910_9391": "ascend910_93", + "ascend910_9381": "ascend910_93", + "ascend910_9372": "ascend910_93", + "ascend910_9392": "ascend910_93", + "ascend910_9382": "ascend910_93", + "ascend910_9361": "ascend910_93", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a", + "ascend610lite": "ascend610lite" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_dtype_for_bin_list = [] + self.input_dtype_for_bin = {} + self.input_fmt = [] + self.input_fmt_for_bin_list = [] + self.input_fmt_for_bin = {} + self.input_virt = {} + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_dtype_for_bin_list = [] + self.output_dtype_for_bin = {} + self.output_fmt = [] + self.output_fmt_for_bin_list = [] + self.output_fmt_for_bin = {} + self.output_init_value = [] + self.output_shape_depend_on_compute = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + self.param_type_dynamic = False + self.mc2_ctx = [] + self.bin_cprs_list = [] + self.bin_cprs_head = [] + self.bin_save_list = [] + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + param_type = self._parse_str(conf) + self.input_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_dtype'.format(int(self.input_idx))): + self.input_dtype_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_format'.format(int(self.input_idx))): + self.input_fmt_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.virtual'.format(int(self.input_idx))): + self.input_virt[self.input_idx] = self._parse_str(conf) + elif conf.startswith('input{}.initValue'.format(int(self.input_idx))): + raise Exception(f'[ERROR]: Op: {{\'{self.op_type}\'}} input {self.input_ori_name[int(self.input_idx)]}\ + has InitValue, which is not support!') + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + self.output_init_value.append(None) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + param_type = self._parse_str(conf) + self.output_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_dtype'.format(int(self.output_idx))): + self.output_dtype_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_format'.format(int(self.output_idx))): + self.output_fmt_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.initValue'.format(int(self.output_idx))): + self.output_init_value[int(self.output_idx)] = self._parse_str(conf) + elif conf.startswith('output{}.outputShapeDependOnCompute=true'.format(int(self.output_idx))): + self.output_shape_depend_on_compute.append(int(self.output_idx)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'[ERROR]: The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} \ +conflicts with the built-in variable name. Use a complex name or prefix the operator name.') + + def parse_mc2_ctx(self: any, conf: str): + self.mc2_ctx = self._parse_list(conf) + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options.update(soc_ver_compile_options) + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + break + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('mc2.ctx'): + op_desc.parse_mc2_ctx(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/parse_ini_to_json.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..52dff4ad3 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/parse_ini_to_json.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights£¬group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + parse_ini_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if not ini_file_path_list: + ini_file_path_list.append("tbe_ops_info.ini") + + for ini_file in ini_file_path_list: + if os.path.exists(ini_file): + parse_ini_list.append(ini_file) + + if parse_ini_list: + if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/preset_parse.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/preset_parse.py new file mode 100755 index 000000000..4ff54c0a7 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/preset_parse.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +import json +import sys +import os + + +def read_json(file): + with open(file, 'r') as fd: + config = json.load(fd) + return config + + +def get_config_opts(file): + config = read_json(file) + + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_codegen.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_impl.temp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/tiling_data_def_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..cb2a01c56 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/cmake/util/tiling_data_def_build.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#undef GET_TILING_DATA +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc new file mode 100644 index 000000000..6b6fc00c7 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("Demo") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Demo") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/CMakeLists.txt new file mode 100644 index 000000000..2b8d9034f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/CMakeLists.txt @@ -0,0 +1,93 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +file(GLOB group_proto_src ${ASCEND_AUTOGEN_PATH}/group_proto/*.cc) + +add_library(cust_op_proto SHARED + $<$:${group_proto_src}> + ${ops_srcs} + ${ASCEND_AUTOGEN_PATH}/op_proto.cc +) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +file(GLOB GROUP_PROTO_HEADERS ${ASCEND_AUTOGEN_PATH}/group_proto/*.h) +if (GROUP_PROTO_HEADERS) + install(FILES ${GROUP_PROTO_HEADERS} + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +endif() +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp new file mode 100644 index 000000000..d8598f558 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp @@ -0,0 +1,66 @@ + +#include "demo_tiling.h" +#include "register/op_def_registry.h" + + +namespace optiling { +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + + DemoTilingData tiling; + uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); + + tiling.set_size(inputNum); + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class Demo : public OpDef { +public: + explicit Demo(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend910b"); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(Demo); +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h new file mode 100644 index 000000000..9dfd12fa5 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -0,0 +1,10 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(DemoTilingData) + TILING_DATA_FIELD_DEF(uint32_t, size); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp new file mode 100644 index 000000000..7184389a5 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -0,0 +1,66 @@ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue +class KernelDemo { +public: + __aicore__ inline KernelDemo() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength) { + this->totalLength = totalLength; + xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); + yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); + zGm.SetGlobalBuffer((__gm__ half *)z, this->totalLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->totalLength * sizeof(half)); + + } + __aicore__ inline void Process() { + CopyIn(); + Compute(); + CopyOut(); + } + +private: + // TODO: user kernel private member + + __aicore__ inline void CopyIn(){ + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); + AscendC::DataCopy(xLocal, xGm, this->totalLength); + AscendC::DataCopy(yLocal, yGm, this->totalLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + + __aicore__ inline void Compute(){ + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = inQueueY.DeQue(); + AscendC::LocalTensor zLocal = outQueueZ.AllocTensor(); + // 预期结果为 x *0.001+y*0.001 = (x + y) * 0.001 + Add(zLocal, xLocal, yLocal, this->totalLength); + Muls(zLocal, zLocal, (half)0.001, this->totalLength); + + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(){ + AscendC::LocalTensor zLocal = outQueueZ.DeQue(); + AscendC::DataCopy(zGm, zLocal, this->totalLength); + outQueueZ.FreeTensor(zLocal); + } +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX,inQueueY; + AscendC::TQue outQueueZ; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + AscendC::GlobalTensor zGm; + uint32_t totalLength; +}; + +extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + KernelDemo op; + op.Init(x, y, z, tiling_data.size); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/install.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/install.sh new file mode 100755 index 000000000..b87a9790f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + exit 1 + fi +fi +chmod u+w ${targetdir}/vendors + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}\nexport LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + log "[ERROR] echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ export LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH} ] to set the environment path" +fi + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/upgrade.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/upgrade.sh new file mode 100755 index 000000000..65457d2dd --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/scripts/upgrade.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + log "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakeLists.txt new file mode 100644 index 000000000..ea2c509e5 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakeLists.txt @@ -0,0 +1,74 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + if(CMAKE_CROSS_LIBRARY_PATH) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + endif() + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakePresets.json b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakePresets.json new file mode 100644 index 000000000..d32bf6d9a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b;ascend910b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/build.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/build.sh new file mode 100755 index 000000000..113fa60d8 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/build.sh @@ -0,0 +1,95 @@ +#!/bin/bash +if [ -z "$BASE_LIBS_PATH" ]; then + if [ -z "$ASCEND_HOME_PATH" ]; then + if [ -z "$ASCEND_AICPU_PATH" ]; then + echo "please set env." + exit 1 + else + export ASCEND_HOME_PATH=$ASCEND_AICPU_PATH + fi + else + export ASCEND_HOME_PATH=$ASCEND_HOME_PATH + fi +else + export ASCEND_HOME_PATH=$BASE_LIBS_PATH +fi +echo "using ASCEND_HOME_PATH: $ASCEND_HOME_PATH" +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/config.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/config.cmake new file mode 100755 index 000000000..72690b9e4 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/config.cmake @@ -0,0 +1,30 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_AUTOGEN_GROUPPROTO_PATH ${CMAKE_BINARY_DIR}/autogen/group_proto) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH} ${ASCEND_AUTOGEN_GROUPPROTO_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + ) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/func.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/func.cmake new file mode 100755 index 000000000..c4da5fee8 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/func.cmake @@ -0,0 +1,212 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + + set(ENV{ENABLE_SOURCE_PACAKGE} ${OPBUILD_ENABLE_SOURCE}) + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + unset(ENV{ENABLE_SOURCE_PACAKGE}) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_gen_options.py + ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} ${OP_TYPE} ${OP_COMPILE_COMPUTE_UNIT} + ${OP_COMPILE_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR) + if (${EXEC_RESULT}) + message("add ops compile options info: ${EXEC_INFO}") + message("add ops compile options error: ${EXEC_ERROR}") + message(FATAL_ERROR "add ops compile options failed!") + endif() +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + --opc-config-file ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + # add Environment Variable Configurations of python & ccache + set(_ASCENDC_ENV_VAR) + list(APPEND _ASCENDC_ENV_VAR export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} &&) + # whether need judging CMAKE_C_COMPILER_LAUNCHER + if(${CMAKE_CXX_COMPILER_LAUNCHER} MATCHES "ccache$") + list(APPEND _ASCENDC_ENV_VAR export ASCENDC_CCACHE_EXECUTABLE=${CMAKE_CXX_COMPILER_LAUNCHER} &&) + endif() + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND ${_ASCENDC_ENV_VAR} bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/intf.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/intf.cmake new file mode 100755 index 000000000..d2643bbc7 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/intf.cmake @@ -0,0 +1,28 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include + ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel +) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/makeself.cmake b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/makeself.cmake new file mode 100755 index 000000000..2c482b042 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/makeself.cmake @@ -0,0 +1,33 @@ +execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) + +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() + +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + +if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") + execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + + execute_process( + COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) +endif() + + diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/__init__.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_bin_param_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..39bfbb13c --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import argparse +import sys +import os +import json +import hashlib +import re +import copy +from collections import defaultdict +from typing import Dict, List, Set, Tuple, NamedTuple + +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ParamInfo(NamedTuple): + dtype_list: list + format_list: list + dtype_for_bin_list: dict + format_for_bin_list: dict + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + self.tiling_keys = set() + self.op_debug_config = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def set_tiling_key(self: any, tiling_key_info: Set): + if tiling_key_info: + self.tiling_keys.update(tiling_key_info) + + def set_op_debug_config(self: any, op_debug_config: str): + if op_debug_config: + self.op_debug_config = op_debug_config + + def get_full_list(self: any): + dtype_list = [] + for dtype_in in self.input_dtype: + dtype_list.append(dtype_in.split(',')) + for dtype_out in self.output_dtype: + dtype_list.append(dtype_out.split(',')) + + format_list = [] + for fmt_in in self.input_fmt: + format_list.append(fmt_in.split(',')) + for fmt_out in self.output_fmt: + format_list.append(fmt_out.split(',')) + + dtype_for_bin_list = [[] for _ in range(len(self.input_dtype) + len(self.output_dtype))] + format_for_bin_list = copy.deepcopy(dtype_for_bin_list) + + for key, value in self.input_dtype_for_bin.items(): + dtype_for_bin_list[key] = value.split(',') + for key, value in self.output_dtype_for_bin.items(): + dtype_for_bin_list[key + len(self.input_dtype)] = value.split(',') + for key, value in self.input_fmt_for_bin.items(): + format_for_bin_list[key] = value.split(',') + for key, value in self.output_fmt_for_bin.items(): + format_for_bin_list[key + len(self.input_dtype)] = value.split(',') + + return ParamInfo(dtype_list, format_list, dtype_for_bin_list, format_for_bin_list) + + + def gen_bin_cprs_list(self: any, param_info: ParamInfo): + combine_dict = {} + origin_combine_dict = {} + for cob_idx in range(0, len(self.input_dtype[0].split(','))): + origin_combine = "" + combine = "" + for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): + if (param_info.dtype_for_bin_list[param_idx]): + combine += param_info.dtype_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.dtype_list[param_idx][cob_idx] + origin_combine += param_info.dtype_list[param_idx][cob_idx] + if (param_info.format_for_bin_list[param_idx]): + combine += param_info.format_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.format_list[param_idx][cob_idx] + origin_combine += param_info.format_list[param_idx][cob_idx] + if (combine not in combine_dict): + combine_dict[combine] = [] + combine_dict[combine].append(cob_idx) + origin_combine_dict[origin_combine] = cob_idx + for key, value in combine_dict.items(): + if (key not in origin_combine_dict): + print(f"WARNING: ForBinQuery {key} not in origin combine") + self.bin_save_list += value + continue + if len(value) == 1 and value[0] == origin_combine_dict[key]: + self.bin_save_list += value + continue + self.bin_cprs_head.append(origin_combine_dict[key]) + self.bin_cprs_list.append(value) + for index, sub_list in enumerate(self.bin_cprs_list): + if self.bin_cprs_head[index] not in self.bin_save_list: + continue + sub_list.append(self.bin_cprs_head[index]) + self.bin_save_list += self.bin_cprs_head + + + def gen_for_bin_list(self: any, param_info: ParamInfo): + combine_size = len(self.input_dtype[0].split(',')) + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + + self.input_dtype_for_bin_list = [[] for _ in range(input_size)] + self.output_dtype_for_bin_list = [[] for _ in range(output_size)] + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) + self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) + + for index, sub_list in enumerate(self.bin_cprs_list): + head_idx = self.bin_cprs_head[index] + for cmb_idx in sub_list: + for i in range(0, input_size): + self.input_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i][cmb_idx]) + self.input_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i][cmb_idx]) + for i in range(0, output_size): + self.output_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i + input_size][cmb_idx]) + self.output_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i + input_size][cmb_idx]) + + + def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.input_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.input_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.input_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.input_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.input_fmt[i] = new_format_str[:-1] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.output_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.output_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.output_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i + input_size]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.output_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i + input_size]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.output_fmt[i] = new_format_str[:-1] + + + def is_set_for_bin_query(self: any): + return any([ + self.input_dtype_for_bin, + self.output_dtype_for_bin, + self.input_fmt_for_bin, + self.output_fmt_for_bin, + ]) + + + def for_bin_list_match(self: any): + if not self.is_set_for_bin_query(): + return + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + param_info = self.get_full_list() + self.gen_bin_cprs_list(param_info) + self.gen_for_bin_list(param_info) + if len(self.bin_save_list) == len(self.input_dtype[0].split(',')): + print(f'WARNING: ForBinQuery can not compress number of bin file with this set, please check!!.') + return + self.rm_cprs_cmb(param_info.dtype_list, param_info.format_list, input_size, output_size) + + + def gen_input_json(self: any, auto_gen_path: str): + key_map = {} + self.for_bin_list_match() + count = len(self.input_dtype[0].split(',')) + required_parameters = set() + index_value = -1 + + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + required_parameter = [] + op_node = {} + + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + if self.is_set_for_bin_query() and self.input_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.input_dtype_for_bin_list[idx][i] + para['format'] = ifmts[i] + if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.input_fmt_for_bin_list[idx][i] + para['paramType'] = itype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + + input_parameter_key = (idtypes[i], ifmts[i]) + if itype == 'dynamic': + inputs.append([para]) + required_parameter.append(input_parameter_key) + elif itype == 'required': + inputs.append(para) + required_parameter.append(input_parameter_key) + else: + inputs.append(para) + + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + if self.is_set_for_bin_query() and self.output_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.output_dtype_for_bin_list[idx][i] + para['format'] = ofmts[i] + if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.output_fmt_for_bin_list[idx][i] + para['paramType'] = otype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + output_parameter_key = (odtypes[i], ofmts[i]) + if otype == 'dynamic': + outputs.append([para]) + required_parameter.append(output_parameter_key) + elif otype == 'required': + outputs.append(para) + required_parameter.append(output_parameter_key) + else: + outputs.append(para) + + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + + required_parameter_tuple = tuple(required_parameter) + if required_parameter_tuple in required_parameters: + continue + else: + required_parameters.add(required_parameter_tuple) + index_value +=1 + + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) + + + def _write_build_cmd(self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str): + hard_soc = const_var.conv_soc_ver(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + + bin_cmd_str = 'res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ + --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic ' + + build_cmd_var = "#!/bin/bash\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' + plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") + plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") + if plog_level is None: + build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR + if plog_stdout is None: + build_cmd_var += const_var.SET_PLOG_STDOUT + build_cmd_var += const_var.SRC_ENV + if hard_soc == "Ascend610Lite": + build_cmd_var += f'export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n' + build_cmd_var += bin_cmd_str.format(fun=self.op_intf, soc=hard_soc, param=param_file, + impl='high_performance,optional') + enable_tiling_keys = False + if self.tiling_keys: + tiling_keys_list = sorted(list(self.tiling_keys)) + tiling_key_str = ','.join([str(_key) for _key in tiling_keys_list]) + build_cmd_var += f' --tiling_key="{tiling_key_str}"' + enable_tiling_keys = True + + if self.op_debug_config: + op_debug_str = ','.join([str(_key) for _key in list(self.op_debug_config)]) + build_cmd_var += f' --op_debug_config={op_debug_str}' + + build_cmd_var += ")\n" + build_cmd_var += "\n" + if enable_tiling_keys is False: + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + else: + build_cmd_var += "if [ $? -eq 1 ]; then\n" + build_cmd_var += " if echo \"${res}\" | \ +grep -q \"None of the given tiling keys are in the supported list\"; then\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " else\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " exit 1\n" + build_cmd_var += " fi\n" + build_cmd_var += "else\n" + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + build_cmd_var += "fi\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' + + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write(build_cmd_var) + + +def get_tiling_keys(tiling_keys: str) -> Set: + all_tiling_keys = set() + if not tiling_keys: + return all_tiling_keys + + tiling_key_list = tiling_keys.split(';') + for tiling_key_value in tiling_key_list: + pattern = r"(? int(end): + continue + for i in range(int(start), int(end) + 1): + all_tiling_keys.add(i) + elif tiling_key_value.isdigit(): + all_tiling_keys.add(int(tiling_key_value)) + return all_tiling_keys + + +def trans_soc_verion(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: + return low_soc_ver + return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: + tiling_key_info = defaultdict(set) + op_debug_config = defaultdict(set) + if not opc_config_file: + return tiling_key_info, op_debug_config + + if not os.path.exists(opc_config_file): + return tiling_key_info, op_debug_config + + with open(opc_config_file, 'r') as file: + contents = file.readlines() + + for _content in contents: + content = _content.strip() + opc_configs = content.split('@') + if len(opc_configs) < 3: + continue + + op_type = opc_configs[0] + if not op_type: + continue + + compute_unit = opc_configs[1] + if compute_unit: + compute_unit_list = compute_unit.split(';') + soc_lists = [] + for soc_ver in compute_unit_list: + short_soc_ver = trans_soc_verion(soc_ver) + soc_lists.append(short_soc_ver) + if soc not in soc_lists: + continue + + for options in opc_configs[2:]: + if "--tiling_key" in options: + format_tiling_keys = get_tiling_keys(options.split('=')[1]) + if format_tiling_keys: + tiling_key_info[op_type].update(format_tiling_keys) + if "--op_debug_config" in options: + format_debug_config = set(options.split("=")[1].split(';')) + if format_debug_config: + op_debug_config[op_type].update(format_debug_config) + + return tiling_key_info, op_debug_config + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str, + opc_config_file: str = '', ops: list = None): + if not os.path.exists(cfgfile): + print(f'INFO: {cfgfile} does not exists in this project, skip generating compile commands.') + return + + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) + tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) + auto_gen_path_dir = os.path.dirname(cfgfile) + all_soc_key = "ALL" + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + if op_desc.op_type in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) + if all_soc_key in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[all_soc_key]) + if op_desc.op_type in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) + if all_soc_key in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[all_soc_key]) + op_desc.gen_input_json(auto_gen_path_dir) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opc-config-file', nargs='?', const='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + if len(args.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(args.argv[1], + args.argv[2], + args.argv[3], + opc_config_file=args.opc_config_file) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_gen_options.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_gen_options.py new file mode 100755 index 000000000..2dd8ee047 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_gen_options.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import sys +import stat +import os +import re +import json +import const_var + + +def write_options_to_file(file_name: str, options_str: str, \ + op_type: str, compute_unit: str, split_char: str): + flags = os.O_WRONLY | os.O_CREAT + modes = stat.S_IWUSR | stat.S_IRUSR + try: + with os.fdopen(os.open(file_name, flags, modes), 'a') as fd: + fd.write(op_type + split_char + compute_unit + split_char + options_str + '\n') + except Exception as err: + print("write compile options config file failed") + raise(err) + + +def gen_compile_options(compile_options_file: str, op_type: str, \ + compute_unit: str, compile_options: list): + base_dir = os.path.dirname(compile_options_file) + opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") + compile_opt = [] + opc_debug_config = [] + opc_tiling_keys = "" + for opts in compile_options: + if "oom" in opts: + if opts == "--oom": + opc_debug_config.append("oom") + else: + raise RuntimeError(f"Unknown oom option format {opts}") + elif "--save-temp-files" in opts: + opc_debug_config.append("dump_cce") + elif "--tiling_key" in opts: + keys = opts.strip().split('=')[1].split(',') + keys_str = ";".join([key for key in keys]) + opc_tiling_keys = keys_str + else: + compile_opt.append(opts) + if len(compile_opt) > 0: + options_str = ';'.join([opt for opt in compile_opt]) + write_options_to_file(compile_options_file, options_str, op_type, compute_unit, ",") + opc_config_str = "" + if opc_debug_config: + opc_config_str = "--op_debug_config=" + ';'.join([opt for opt in opc_debug_config]) + if len(opc_tiling_keys) > 0: + if opc_config_str != "": + opc_config_str += "@" + opc_config_str += "--tiling_key=" + opc_tiling_keys + + if opc_config_str != "": + write_options_to_file(opc_config_file, opc_config_str, op_type, compute_unit, "@") + + +if __name__ == '__main__': + if len(sys.argv) < 4: + raise RuntimeError('arguments must greater than 4') + compute_soc = "" + comp_options = [] + for i in range(len(sys.argv) - 3): + if sys.argv[i + 3].upper().startswith("ASCEND"): + compute_soc += sys.argv[i + 3] + ";" + else: + comp_options.append(sys.argv[i + 3]) + if compute_soc != "": + compute_soc = compute_soc[0:-1] + gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_impl_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..caa3ab149 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_impl_build.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" + +import argparse +import glob +import sys +import os +import re +import datetime +from typing import List + +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = '''#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. +""" + +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex32": ["DT_COMPLEX32", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) + raise RuntimeError("cannot open %s" %(so_path)) + else: + msg = "load so succ " + so_path + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options.extend(compile_option_list[shortsoc]) + if '__ALLSOC__' in compile_option_list: + compile_options.extend(compile_option_list['__ALLSOC__']) + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): +{} + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = os.environ.get('BISHENG_REAL_PATH') + if bisheng is None: + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_current_build_config("enable_deterministic_mode") == 1: + options.append("-DDETERMINISTIC_MODE=1") + else: + options.append("-DDETERMINISTIC_MODE=0") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + msg = "start replay Acend C Operator {}, kernel name is {}" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API_BUILT_IN = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + + op_compile_option = '{}' + opp_path = os.environ.get('ASCEND_OPP_PATH') + dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) + if opp_path and os.path.exists(dat_path): + # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op + abs_rel_kernel_src_path = "{}" + extend_options = {{}} + extend_options['opp_kernel_hidden_dat_path'] = dat_path + compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ + extend_options) + else: + raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code == 0: + sup = "True" + reason = "" + elif err_code == 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + snake_case += "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + snake_case += "_" + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all: list = None): + self._build_paradefault() + if os.environ.get('BUILD_BUILTIN_OPP') != '1' and impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + print(f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check.") + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd, impl_path) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self: any, op_compile_option_all: list = None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format('OPTION', att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_virtual(self: any) -> str: + virt_exp = [] + for index in range(len(self.input_name)): + if self.input_virt.get(index) is None: + continue + val = [] + val.append('"param_name":"{}"'.format(self.input_name[index])) + val.append('"index":{}'.format(index)) + val.append('"dtype":"{}"'.format(self.input_dtype[index].split(',')[0])) + val.append('"format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"paramType":"optional"') + val.append('"shape":[1]') + val.append('"ori_shape":[1]') + virt_exp.append(' ' + self.input_name[index] + ' = {' + ','.join(val) + '}') + if len(virt_exp) > 0: + return '\n'.join(virt_exp) + else: + return ' # do ascendc build step' + + def _build_mc2_ctx(self: any): + if len(self.mc2_ctx) != 0: + return '["' + '", "'.join(self.mc2_ctx) + '"]' + return '[]' + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + in_idx = 0 + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + in_idx += 1 + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + now = datetime.datetime.now() + curr_year = now.year + former_year = curr_year - 1 + fd.write(IMPL_HEAD.format(former_year, curr_year, self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): + src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + + def _write_impl(self: any, fd: object, impl_path: str = ""): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + virt_exprs = self._build_virtual() + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, virt_exprs, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file,\ + self.op_type, self.op_file, self.param_type_dynamic, self.op_compile_option)) + else: + if os.environ.get('BUILD_BUILTIN_OPP') == '1': + relative_kernel_src_path = os.path.realpath(self._get_kernel_source(impl_path, src,\ + optype_snake(self.op_type), optype_snake_ex(self.op_type))) + # to match src path in .dat file system, turn relative path into absolute path + abs_rel_kernel_src_path = os.path.join("/", os.path.relpath(relative_kernel_src_path, impl_path)) + + # compiling hidden src file requires src path before packaging .dat file, + # hard code such src path to .py + fd.write(COMPILE_OP_API_BUILT_IN.format(self.op_type, self.op_type,\ + ', '.join(self.input_name), ', '.join(self.output_name), self.param_type_dynamic,\ + self._build_mc2_ctx(), self.input_type + self.output_type, self.output_init_value,\ + self.output_shape_depend_on_compute, self.op_compile_option, abs_rel_kernel_src_path)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.param_type_dynamic, self._build_mc2_ctx(),\ + self.input_type + self.output_type, self.output_init_value, self.output_shape_depend_on_compute,\ + self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + + +class OpFileNotExistsError(Exception): + """File does not exist error.""" + def __str__(self) -> str: + return f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" + + +def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: + """Get all ops info files.""" + ops_info_files = [] + for _dir in opsinfo_dir: + ops_info_files.extend(glob.glob(f'{_dir}/aic-*-ops-info.ini')) + return sorted(ops_info_files) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opsinfo-dir', nargs='*', default=None) + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + + if len(args.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] + + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] + + ops_infos = [] + if args.opsinfo_dir: + ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) + if not ops_infos: + raise OpFileNotExistsError(args.opsinfo_dir) + else: + ops_infos.append(args.argv[1]) + + for ops_info in ops_infos: + write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_ops_config.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..8147f9139 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_ops_config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. +""" + +import os +import glob +import json +import sys +import argparse +import const_var + + +BINARY_INFO_CONFIG_JSON = "binary_info_config.json" + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return sorted(all_suffix_files) + + +def add_dict_key(dict_to_add, key, value): + if value is None: + return + dict_to_add[key] = value + + +def correct_format_mode(format_mode): + if format_mode == 'FormatDefault': + return 'nd_agnostic' + if format_mode == 'FormatAgnostic': + return 'static_nd_agnostic' + if format_mode == 'FormatFixed': + return 'normal' + return format_mode + + +def get_input_or_output_config(in_or_out): + param_dict = {} + name = in_or_out.get('name') + index = in_or_out.get('index') + param_type = in_or_out.get('paramType') + + format_match_mode = in_or_out.get('format_match_mode') + format_mode = correct_format_mode(format_match_mode) + + dtype_mode = in_or_out.get('dtype_match_mode') + if dtype_mode == 'DtypeByte': + dtype_mode = 'bit' + + add_dict_key(param_dict, 'name', name) + add_dict_key(param_dict, 'index', index) + add_dict_key(param_dict, 'paramType', param_type) + add_dict_key(param_dict, 'dtypeMode', dtype_mode) + add_dict_key(param_dict, 'formatMode', format_mode) + return param_dict + + +def get_inputs_or_outputs_config(inputs_or_outputs): + if inputs_or_outputs is None: + return None + inputs_or_outputs_list = [] + + for in_or_out in inputs_or_outputs: + if isinstance(in_or_out, dict): + dict_param_config = get_input_or_output_config(in_or_out) + inputs_or_outputs_list.append(dict_param_config) + elif isinstance(in_or_out, list): + param_info = in_or_out[0] + list_param_config = get_input_or_output_config(param_info) + tmp_list = [list_param_config] + inputs_or_outputs_list.append(tmp_list) + return inputs_or_outputs_list + + +def gen_attrs_config(attrs): + attrs_list = [] + for attr in attrs: + attrs_dict = {} + name = attr.get('name') + mode = attr.get('mode') + add_dict_key(attrs_dict, 'name', name) + add_dict_key(attrs_dict, 'mode', mode) + attrs_list.append(attrs_dict) + return attrs_list + + +def get_params_config(support_info): + params_dict = {} + + inputs = support_info.get('inputs') + inputs_list = get_inputs_or_outputs_config(inputs) + params_dict['inputs'] = inputs_list + + outputs = support_info.get('outputs') + outputs_list = get_inputs_or_outputs_config(outputs) + params_dict['outputs'] = outputs_list + + attrs = support_info.get('attrs') + if attrs is not None: + attrs_list = gen_attrs_config(attrs) + params_dict['attrs'] = attrs_list + + return params_dict + + +def add_simplified_config(op_type, support_info, core_type, task_ration, objfile, config): + simplified_key = support_info.get('simplifiedKey') + + json_path = objfile.split('.')[0] + '.json' + + simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {'dynamicRankSupport': True} + + simplified_key_mode = support_info.get('simplifiedKeyMode') + add_dict_key(op_cfg, 'simplifiedKeyMode', simplified_key_mode) + + optional_input_mode = support_info.get('optionalInputMode') + optional_output_mode = support_info.get('optionalOutputMode') + add_dict_key(op_cfg, 'optionalInputMode', optional_input_mode) + if optional_output_mode is not None: + add_dict_key(op_cfg, 'optionalOutputMode', optional_output_mode) + + params_info = get_params_config(support_info) + op_cfg['params'] = params_info + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'multiKernelType': 1, 'binPath': objfile, 'jsonPath': json_path}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'binPath': objfile, 'jsonPath': json_path}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {'binList': []} + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE': 3, 'MIX_VECTOR_CORE': 4, 'MIX_AIV': 4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = contents.get("coreType") + task_ration = contents.get("taskRation") + core_type = core_type_map.get(core_type, -1) + if core_type == -1 and soc != 'ascend310b': + raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") + + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + + add_dict_key(bin_info, 'implMode', support_info.get('implMode')) + add_dict_key(bin_info, 'int64Mode', support_info.get('int64Mode')) + add_dict_key(bin_info, 'simplifiedKeyMode', support_info.get('simplifiedKeyMode')) + + simplified_key = support_info.get('simplifiedKey') + if simplified_key is not None: + bin_info['simplifiedKey'] = simplified_key + obj_file = os.path.join(soc, op_dir, bin_file_name) + add_simplified_config(op_type, support_info, core_type, task_ration, obj_file, config) + + add_dict_key(bin_info, 'dynamicParamMode', support_info.get('dynamicParamMode')) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + + add_dict_key(bin_info, 'opMode', support_info.get('opMode')) + add_dict_key(bin_info, 'optionalInputMode', support_info.get('optionalInputMode')) + add_dict_key(bin_info, 'deterministic', support_info.get('deterministic')) + if support_info.get('optionalOutputMode') is not None: + add_dict_key(bin_info, 'optionalOutputMode', support_info.get('optionalOutputMode')) + + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def check_single_op_is_void(root_dir): + for root, dirs, _ in os.walk(root_dir): + for sub_dir in dirs: + dir_path = os.path.join(root, sub_dir) + if len(os.listdir(dir_path)) == 0: + print(f"[ERROR] op {sub_dir}: not any obj compile success") + sys.exit(1) + + +def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): + suffix = 'json' + config = {BINARY_INFO_CONFIG_JSON: {}} + check_single_op_is_void(root_dir) + all_json_files = get_specified_suffix_file(root_dir, suffix) + + for _json in all_json_files: + gen_ops_config(_json, soc, config) + file_path = soc + _json.split(soc)[1] + with open(_json, "r+") as f: + data = json.load(f) + data["filePath"] = file_path + f.seek(0) + json.dump(data, f, indent=" ") + f.truncate() + + for cfg_key in config.keys(): + if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: + continue + cfg_file = os.path.join(out_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +# Parse multiple soc_versions ops in single path. +def gen_all_soc_config(all_path): + soc_roots = glob.glob(os.path.join(all_path, "ascend*")) + + for soc_root in soc_roots: + soc = os.path.basename(soc_root) + gen_all_config(soc_root, soc, soc_root, True) + cfg_files = glob.glob(os.path.join(soc_root, "*.json")) + cfg_path = os.path.join(all_path, "config", soc) + os.makedirs(cfg_path, exist_ok=True) + for cfg_file in cfg_files: + new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) + os.rename(cfg_file, new_file) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + + parser.add_argument('-o', + '--out', + nargs='?', + help='Output directory.') + + parser.add_argument('--skip-binary-info-config', + action='store_true', + help='binary_info_config.json file is not parsed.') + + return parser.parse_args() + + +def main(): + args = args_prase() + if args.out is None: + out_dir = args.path + else: + out_dir = args.out + + gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_replay_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/batch_replay_impl.temp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/code_channel_infer.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..3d1e4e87a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product == "ascend910_93" or op_product == "ascend910b": + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/const_var.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/const_var.py new file mode 100755 index 000000000..c7c41d9dd --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/const_var.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', + 'ascend910_93': 'Ascend910_9391', 'ascend610lite': 'Ascend610Lite'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" +SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" +SRC_ENV = ''' +while true; do + case "$1" in + --kernel-src=*) + export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) + shift + ;; + -*) + shift + ;; + *) + break + ;; + esac +done +''' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} + + +def conv_soc_ver(ver: str): + return SOC_MAP_EXT.get(ver) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..b49cf8b9b --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_ops_filter.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_version_info.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..5144edb1c --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/gen_version_info.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_op_info.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_simplified_keys.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..76f8ad286 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/insert_simplified_keys.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_entry.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_impl.temp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/COPYING b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/README.md b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/VERSION b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/make-release.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself-header.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.1 b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.lsm b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/run-tests.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..8576d7616 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/opdesc_parser.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..fa7002b7a --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/opdesc_parser.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910b4-1": "ascend910b", + "ascend910_9391": "ascend910_93", + "ascend910_9381": "ascend910_93", + "ascend910_9372": "ascend910_93", + "ascend910_9392": "ascend910_93", + "ascend910_9382": "ascend910_93", + "ascend910_9361": "ascend910_93", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a", + "ascend610lite": "ascend610lite" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_dtype_for_bin_list = [] + self.input_dtype_for_bin = {} + self.input_fmt = [] + self.input_fmt_for_bin_list = [] + self.input_fmt_for_bin = {} + self.input_virt = {} + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_dtype_for_bin_list = [] + self.output_dtype_for_bin = {} + self.output_fmt = [] + self.output_fmt_for_bin_list = [] + self.output_fmt_for_bin = {} + self.output_init_value = [] + self.output_shape_depend_on_compute = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + self.param_type_dynamic = False + self.mc2_ctx = [] + self.bin_cprs_list = [] + self.bin_cprs_head = [] + self.bin_save_list = [] + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + param_type = self._parse_str(conf) + self.input_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_dtype'.format(int(self.input_idx))): + self.input_dtype_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_format'.format(int(self.input_idx))): + self.input_fmt_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.virtual'.format(int(self.input_idx))): + self.input_virt[self.input_idx] = self._parse_str(conf) + elif conf.startswith('input{}.initValue'.format(int(self.input_idx))): + raise Exception(f'[ERROR]: Op: {{\'{self.op_type}\'}} input {self.input_ori_name[int(self.input_idx)]}\ + has InitValue, which is not support!') + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + self.output_init_value.append(None) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + param_type = self._parse_str(conf) + self.output_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_dtype'.format(int(self.output_idx))): + self.output_dtype_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_format'.format(int(self.output_idx))): + self.output_fmt_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.initValue'.format(int(self.output_idx))): + self.output_init_value[int(self.output_idx)] = self._parse_str(conf) + elif conf.startswith('output{}.outputShapeDependOnCompute=true'.format(int(self.output_idx))): + self.output_shape_depend_on_compute.append(int(self.output_idx)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'[ERROR]: The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} \ +conflicts with the built-in variable name. Use a complex name or prefix the operator name.') + + def parse_mc2_ctx(self: any, conf: str): + self.mc2_ctx = self._parse_list(conf) + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options.update(soc_ver_compile_options) + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + break + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('mc2.ctx'): + op_desc.parse_mc2_ctx(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/parse_ini_to_json.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..52dff4ad3 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/parse_ini_to_json.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights£¬group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + parse_ini_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if not ini_file_path_list: + ini_file_path_list.append("tbe_ops_info.ini") + + for ini_file in ini_file_path_list: + if os.path.exists(ini_file): + parse_ini_list.append(ini_file) + + if parse_ini_list: + if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/preset_parse.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/preset_parse.py new file mode 100755 index 000000000..4ff54c0a7 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/preset_parse.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +import json +import sys +import os + + +def read_json(file): + with open(file, 'r') as fd: + config = json.load(fd) + return config + + +def get_config_opts(file): + config = read_json(file) + + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_codegen.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_impl.temp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/tiling_data_def_build.py b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..cb2a01c56 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/cmake/util/tiling_data_def_build.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#undef GET_TILING_DATA +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc new file mode 100644 index 000000000..6b6fc00c7 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("Demo") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Demo") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/CMakeLists.txt new file mode 100644 index 000000000..2b8d9034f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/CMakeLists.txt @@ -0,0 +1,93 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +file(GLOB group_proto_src ${ASCEND_AUTOGEN_PATH}/group_proto/*.cc) + +add_library(cust_op_proto SHARED + $<$:${group_proto_src}> + ${ops_srcs} + ${ASCEND_AUTOGEN_PATH}/op_proto.cc +) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +file(GLOB GROUP_PROTO_HEADERS ${ASCEND_AUTOGEN_PATH}/group_proto/*.h) +if (GROUP_PROTO_HEADERS) + install(FILES ${GROUP_PROTO_HEADERS} + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +endif() +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp new file mode 100644 index 000000000..d8598f558 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp @@ -0,0 +1,66 @@ + +#include "demo_tiling.h" +#include "register/op_def_registry.h" + + +namespace optiling { +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + + DemoTilingData tiling; + uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); + + tiling.set_size(inputNum); + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class Demo : public OpDef { +public: + explicit Demo(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("z") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend910b"); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(Demo); +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h new file mode 100644 index 000000000..9dfd12fa5 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -0,0 +1,10 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(DemoTilingData) + TILING_DATA_FIELD_DEF(uint32_t, size); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) +} diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/CMakeLists.txt b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp new file mode 100644 index 000000000..221124607 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -0,0 +1,66 @@ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue +class KernelDemo { +public: + __aicore__ inline KernelDemo() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, GM_ADDR z, uint32_t totalLength) { + this->totalLength = totalLength; + xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); + yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); + zGm.SetGlobalBuffer((__gm__ half *)z, this->totalLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->totalLength * sizeof(half)); + + } + __aicore__ inline void Process() { + CopyIn(); + Compute(); + CopyOut(); + } + +private: + // TODO: user kernel private member + + __aicore__ inline void CopyIn(){ + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); + AscendC::DataCopy(xLocal, xGm, this->totalLength); + AscendC::DataCopy(yLocal, yGm, this->totalLength); + inQueueX.EnQue(xLocal); + inQueueY.EnQue(yLocal); + } + + __aicore__ inline void Compute(){ + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = inQueueY.DeQue(); + AscendC::LocalTensor zLocal = outQueueZ.AllocTensor(); + Muls(xLocal, xLocal, (half)0.001, this->totalLength); + Muls(yLocal, yLocal, (half)0.001, this->totalLength); + Add(zLocal, xLocal, yLocal, this->totalLength); + + outQueueZ.EnQue(zLocal); + inQueueX.FreeTensor(xLocal); + inQueueY.FreeTensor(yLocal); + } + __aicore__ inline void CopyOut(){ + AscendC::LocalTensor zLocal = outQueueZ.DeQue(); + AscendC::DataCopy(zGm, zLocal, this->totalLength); + outQueueZ.FreeTensor(zLocal); + } +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX,inQueueY; + AscendC::TQue outQueueZ; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + AscendC::GlobalTensor zGm; + uint32_t totalLength; +}; + +extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + KernelDemo op; + op.Init(x, y, z, tiling_data.size); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/install.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/install.sh new file mode 100755 index 000000000..b87a9790f --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + exit 1 + fi +fi +chmod u+w ${targetdir}/vendors + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}\nexport LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + log "[ERROR] echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ export LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH} ] to set the environment path" +fi + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/upgrade.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/upgrade.sh new file mode 100755 index 000000000..65457d2dd --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/scripts/upgrade.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + log "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/README.md b/operator_contrib/PrecisionTips/SequenceOperationSteps/README.md new file mode 100644 index 000000000..86f05b959 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/README.md @@ -0,0 +1,157 @@ +## 概述 +该样例用于复现强制转换前后顺序导致的精度损失问题。 +## 目录结构介绍 +``` +├── TypeConversion //使用框架调用的方式调用Demo算子 +│ ├── AclNNInvocation // 通过aclnn调用的方式调用Demo算子 +│ ├── Demo // Demo精度异常算子工程 +│ ├── DemoFix // Demo精度正常算子工程 +│ └── Demo.json // Demo算子的原型定义json文件 +``` +## 算子工程介绍 +算子工程目录Demo包含算子实现的模板文件、编译脚本等，如下所示: +``` +├── Demo //Demo自定义算子工程 +│ ├── cmake +│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 +│ ├── op_host // host侧实现文件 +│ ├── op_kernel // kernel侧实现文件 +│ ├── scripts // 自定义算子工程打包相关脚本所在目录 +│ ├── build.sh // 编译入口脚本 +│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt +│ └── CMakePresets.json // 编译配置项 +``` +CANN软件包中提供了工程创建工具msopgen，Demo算子工程可通过Demo.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 +## 编译运行样例算子 +针对自定义算子工程，编译运行包含如下步骤： +- 编译自定义算子工程生成算子安装包； +- 安装自定义算子到算子库中； +- 调用执行自定义算子； + +详细操作如下所示。 +### 1. 获取源码包 +请参考[准备：获取样例代码](../README.md#codeready)完成源码包获取。 +### 2. 修改算子工程配置 + + + - 执行如下命令，切换到算子工程Demo目录。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo + ``` + + - 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + + - 执行如下命令，切换到算子工程DemoFix目录。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix + ``` + + - 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + + +### 3. 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` +### 4. 调用执行算子工程 + +运行run_test.sh，即可复现错误，界面会输出 + ``` +*************** run Demo start**************** +生成测试数据 +[ 86.94 102.5 71.6 ... 102.5 86.9 81. ] +INFO: generate input data success! +可执行不存在 +INFO: cmake success! +INFO: make success! +INFO: execute op! +INFO: acl executable run success! +first 10 mistakes data index: 000000, expected: 86.937500000, actual: inf, rdiff: inf +first 10 mistakes data index: 000001, expected: 102.500000000, actual: inf, rdiff: inf +first 10 mistakes data index: 000002, expected: 71.625000000, actual: inf, rdiff: inf +first 10 mistakes data index: 000004, expected: 93.000000000, actual: inf, rdiff: inf +first 10 mistakes data index: 000008, expected: 125.000000000, actual: inf, rdiff: inf +first 10 mistakes data index: 000009, expected: 88.375000000, actual: inf, rdiff: inf +first 10 mistakes data index: 000010, expected: 99.062500000, actual: inf, rdiff: inf +first 10 mistakes data index: 000013, expected: 78.062500000, actual: inf, rdiff: inf +first 10 mistakes data index: 000014, expected: 12.984375000, actual: 13.000000000, rdiff: 0.001204 +first 10 mistakes data index: 000017, expected: 75.312500000, actual: inf, rdiff: inf +first 10 mistakes data index: 000018, expected: 92.062500000, actual: inf, rdiff: inf +error ratio: 0.5098, tolrence: 0.0010 +[ERROR] result error +*************** run Demo end**************** +*************** run DemoFix start**************** +已存在测试数据 +INFO: generate input data success! +可执行存在 +INFO: execute op! +INFO: acl executable run success! +error ratio: 0.0000, tolrence: 0.0010 +test pass +*************** run DemoFix end**************** + +``` + +## 精度调优思路 +修改前代码: +``` + // 预期结果为 x *0.001+y*0.001 = (x + y) * 0.001 + Add(zLocal, xLocal, yLocal, this->totalLength); + Muls(zLocal, zLocal, (half)0.001, this->totalLength); +``` +预期结果为 x *0.001 + y*0.001 ，该运算可以简化为(x + y) * 0.001,但是x+y 可能会超出float16数据类型表达范围，导致结果计算错误。 + +修改后代码: +``` + Muls(xLocal, xLocal, (half)0.001, this->totalLength); + Muls(yLocal, yLocal, (half)0.001, this->totalLength); + Add(zLocal, xLocal, yLocal, this->totalLength); +``` +修改后代码运算未简化，按照标杆运算步骤进行运算，精度符合预期 diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh new file mode 100755 index 000000000..467a73309 --- /dev/null +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +LOG_PATH=/var/log/npu/slog/debug +if [ -d "$LOG_PATH" ]; then + echo "清理日志，路径：$LOG_PATH" +else + LOG_PATH=~/ascend/log + echo "清理日志，路径：$LOG_PATH" +fi +rm -rf $LOG_PATH* + +current_path=$(pwd) +echo "*************** run Demo start****************" +cd Demo;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 +cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh +echo "*************** run Demo end****************" +cd $current_path +echo "*************** run DemoFix start****************" +cd DemoFix;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 +cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh +echo "*************** run DemoFix end****************" diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/common.h b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/op_runner.h b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/operator_desc.h b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..17607353d --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,60 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + bool keepDims; + bool ignoreNan; + char *dtype; + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/run.sh b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/run.sh new file mode 100644 index 000000000..98c688714 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/run.sh @@ -0,0 +1,109 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + # rm ./input/*.bin + rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_op >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + python3 scripts/verify_result.py output/output.bin output/golden.bin + +} + +main diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/acl.json b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/gen_data.py b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..9c469d9f6 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import numpy as np +import os +np.random.seed(0) +def gen_golden_data_simple(): + input_x = np.random.uniform(1, 65504 / 2048, [2048]).astype(np.float16) + golden = np.random.uniform(0, 0, [2048]).astype(np.float16) + for i in range(2048): + tmp = input_x[:i+1] + golden[i] = np.sum(tmp).astype(np.float16) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/verify_result.py b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..3b5751e1f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== + +import sys +import numpy as np + +# for float16 +relative_tol = 1e-3 +absolute_tol = 1e-5 +error_tol = 1e-3 + + +def verify_result(output, golden): + output = np.fromfile(output, dtype=np.float16).reshape(-1) + golden = np.fromfile(golden, dtype=np.float16).reshape(-1) + different_element_results = np.isclose(output, + golden, + rtol=relative_tol, + atol=absolute_tol, + equal_nan=True) + different_element_indexes = np.where(different_element_results == False)[0] + for index in range(len(different_element_indexes)): + real_index = different_element_indexes[index] + golden_data = golden[real_index] + output_data = output[real_index] + print( + "first 10 mistakes data index: %06d, expected: %-.9f, actual: %-.9f, rdiff: %-.6f" % + (real_index, golden_data, output_data, + abs(output_data - golden_data) / golden_data)) + if index == 10: + break + error_ratio = float(different_element_indexes.size) / golden.size + print("error ratio: %.4f, tolrence: %.4f" % (error_ratio, error_tol)) + return error_ratio <= error_tol + + +if __name__ == '__main__': + try: + res = verify_result(sys.argv[1], sys.argv[2]) + if not res: + raise ValueError("[ERROR] result error") + else: + print("test pass") + except Exception as e: + print(e) + sys.exit(1) diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..ee73e930f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include + ${BUILT_PKG_PATH}/include/aclnnop +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/main.cpp b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..081e72b84 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/main.cpp @@ -0,0 +1,163 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape{2048}; + std::vector shape_out{2048}; + aclDataType dataType = ACL_FLOAT16; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape_out.size(), shape_out.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..2934f4cae --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,485 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_demo.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto ret = aclDestroyTensor(inputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(inputBuffers_[i]); + + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devInputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostInputs_[i]); + } else { + ret = aclrtFreeHost(hostInputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto ret = aclDestroyTensor(outputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(outputBuffers_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devOutputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostOutputs_[i]); + } else { + ret = aclrtFreeHost(hostOutputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + + auto ret = aclnnDemoGetWorkspaceSize(inputTensor_[0], outputTensor_[0], &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + ret = aclnnDemo(workspace, workspaceSize, handle, stream); + + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute Operator success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo.json b/operator_contrib/PrecisionTips/TypeConversion/Demo.json new file mode 100644 index 000000000..bc7789518 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo.json @@ -0,0 +1,22 @@ +[ + { + "op": "Demo", + "language": "cpp", + "input_desc": [ + { + "name": "x", + "param_type": "required", + "format": ["ND"], + "type": ["float16"] + } + ], + "output_desc": [ + { + "name": "y", + "param_type": "required", + "format": ["ND"], + "type": ["float16"] + } + ] + } +] \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/Demo/CMakeLists.txt new file mode 100644 index 000000000..ea2c509e5 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/CMakeLists.txt @@ -0,0 +1,74 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + if(CMAKE_CROSS_LIBRARY_PATH) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + endif() + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/CMakePresets.json b/operator_contrib/PrecisionTips/TypeConversion/Demo/CMakePresets.json new file mode 100644 index 000000000..d32bf6d9a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b;ascend910b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/build.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/build.sh new file mode 100755 index 000000000..113fa60d8 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/build.sh @@ -0,0 +1,95 @@ +#!/bin/bash +if [ -z "$BASE_LIBS_PATH" ]; then + if [ -z "$ASCEND_HOME_PATH" ]; then + if [ -z "$ASCEND_AICPU_PATH" ]; then + echo "please set env." + exit 1 + else + export ASCEND_HOME_PATH=$ASCEND_AICPU_PATH + fi + else + export ASCEND_HOME_PATH=$ASCEND_HOME_PATH + fi +else + export ASCEND_HOME_PATH=$BASE_LIBS_PATH +fi +echo "using ASCEND_HOME_PATH: $ASCEND_HOME_PATH" +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/config.cmake b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/config.cmake new file mode 100755 index 000000000..72690b9e4 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/config.cmake @@ -0,0 +1,30 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_AUTOGEN_GROUPPROTO_PATH ${CMAKE_BINARY_DIR}/autogen/group_proto) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH} ${ASCEND_AUTOGEN_GROUPPROTO_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + ) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/func.cmake b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/func.cmake new file mode 100755 index 000000000..c4da5fee8 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/func.cmake @@ -0,0 +1,212 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + + set(ENV{ENABLE_SOURCE_PACAKGE} ${OPBUILD_ENABLE_SOURCE}) + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + unset(ENV{ENABLE_SOURCE_PACAKGE}) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_gen_options.py + ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} ${OP_TYPE} ${OP_COMPILE_COMPUTE_UNIT} + ${OP_COMPILE_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR) + if (${EXEC_RESULT}) + message("add ops compile options info: ${EXEC_INFO}") + message("add ops compile options error: ${EXEC_ERROR}") + message(FATAL_ERROR "add ops compile options failed!") + endif() +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + --opc-config-file ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + # add Environment Variable Configurations of python & ccache + set(_ASCENDC_ENV_VAR) + list(APPEND _ASCENDC_ENV_VAR export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} &&) + # whether need judging CMAKE_C_COMPILER_LAUNCHER + if(${CMAKE_CXX_COMPILER_LAUNCHER} MATCHES "ccache$") + list(APPEND _ASCENDC_ENV_VAR export ASCENDC_CCACHE_EXECUTABLE=${CMAKE_CXX_COMPILER_LAUNCHER} &&) + endif() + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND ${_ASCENDC_ENV_VAR} bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/intf.cmake b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/intf.cmake new file mode 100755 index 000000000..d2643bbc7 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/intf.cmake @@ -0,0 +1,28 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include + ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel +) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/makeself.cmake b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/makeself.cmake new file mode 100755 index 000000000..2c482b042 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/makeself.cmake @@ -0,0 +1,33 @@ +execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) + +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() + +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + +if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") + execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + + execute_process( + COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) +endif() + + diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/__init__.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_bin_param_build.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..39bfbb13c --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import argparse +import sys +import os +import json +import hashlib +import re +import copy +from collections import defaultdict +from typing import Dict, List, Set, Tuple, NamedTuple + +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ParamInfo(NamedTuple): + dtype_list: list + format_list: list + dtype_for_bin_list: dict + format_for_bin_list: dict + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + self.tiling_keys = set() + self.op_debug_config = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def set_tiling_key(self: any, tiling_key_info: Set): + if tiling_key_info: + self.tiling_keys.update(tiling_key_info) + + def set_op_debug_config(self: any, op_debug_config: str): + if op_debug_config: + self.op_debug_config = op_debug_config + + def get_full_list(self: any): + dtype_list = [] + for dtype_in in self.input_dtype: + dtype_list.append(dtype_in.split(',')) + for dtype_out in self.output_dtype: + dtype_list.append(dtype_out.split(',')) + + format_list = [] + for fmt_in in self.input_fmt: + format_list.append(fmt_in.split(',')) + for fmt_out in self.output_fmt: + format_list.append(fmt_out.split(',')) + + dtype_for_bin_list = [[] for _ in range(len(self.input_dtype) + len(self.output_dtype))] + format_for_bin_list = copy.deepcopy(dtype_for_bin_list) + + for key, value in self.input_dtype_for_bin.items(): + dtype_for_bin_list[key] = value.split(',') + for key, value in self.output_dtype_for_bin.items(): + dtype_for_bin_list[key + len(self.input_dtype)] = value.split(',') + for key, value in self.input_fmt_for_bin.items(): + format_for_bin_list[key] = value.split(',') + for key, value in self.output_fmt_for_bin.items(): + format_for_bin_list[key + len(self.input_dtype)] = value.split(',') + + return ParamInfo(dtype_list, format_list, dtype_for_bin_list, format_for_bin_list) + + + def gen_bin_cprs_list(self: any, param_info: ParamInfo): + combine_dict = {} + origin_combine_dict = {} + for cob_idx in range(0, len(self.input_dtype[0].split(','))): + origin_combine = "" + combine = "" + for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): + if (param_info.dtype_for_bin_list[param_idx]): + combine += param_info.dtype_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.dtype_list[param_idx][cob_idx] + origin_combine += param_info.dtype_list[param_idx][cob_idx] + if (param_info.format_for_bin_list[param_idx]): + combine += param_info.format_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.format_list[param_idx][cob_idx] + origin_combine += param_info.format_list[param_idx][cob_idx] + if (combine not in combine_dict): + combine_dict[combine] = [] + combine_dict[combine].append(cob_idx) + origin_combine_dict[origin_combine] = cob_idx + for key, value in combine_dict.items(): + if (key not in origin_combine_dict): + print(f"WARNING: ForBinQuery {key} not in origin combine") + self.bin_save_list += value + continue + if len(value) == 1 and value[0] == origin_combine_dict[key]: + self.bin_save_list += value + continue + self.bin_cprs_head.append(origin_combine_dict[key]) + self.bin_cprs_list.append(value) + for index, sub_list in enumerate(self.bin_cprs_list): + if self.bin_cprs_head[index] not in self.bin_save_list: + continue + sub_list.append(self.bin_cprs_head[index]) + self.bin_save_list += self.bin_cprs_head + + + def gen_for_bin_list(self: any, param_info: ParamInfo): + combine_size = len(self.input_dtype[0].split(',')) + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + + self.input_dtype_for_bin_list = [[] for _ in range(input_size)] + self.output_dtype_for_bin_list = [[] for _ in range(output_size)] + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) + self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) + + for index, sub_list in enumerate(self.bin_cprs_list): + head_idx = self.bin_cprs_head[index] + for cmb_idx in sub_list: + for i in range(0, input_size): + self.input_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i][cmb_idx]) + self.input_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i][cmb_idx]) + for i in range(0, output_size): + self.output_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i + input_size][cmb_idx]) + self.output_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i + input_size][cmb_idx]) + + + def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.input_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.input_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.input_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.input_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.input_fmt[i] = new_format_str[:-1] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.output_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.output_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.output_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i + input_size]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.output_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i + input_size]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.output_fmt[i] = new_format_str[:-1] + + + def is_set_for_bin_query(self: any): + return any([ + self.input_dtype_for_bin, + self.output_dtype_for_bin, + self.input_fmt_for_bin, + self.output_fmt_for_bin, + ]) + + + def for_bin_list_match(self: any): + if not self.is_set_for_bin_query(): + return + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + param_info = self.get_full_list() + self.gen_bin_cprs_list(param_info) + self.gen_for_bin_list(param_info) + if len(self.bin_save_list) == len(self.input_dtype[0].split(',')): + print(f'WARNING: ForBinQuery can not compress number of bin file with this set, please check!!.') + return + self.rm_cprs_cmb(param_info.dtype_list, param_info.format_list, input_size, output_size) + + + def gen_input_json(self: any, auto_gen_path: str): + key_map = {} + self.for_bin_list_match() + count = len(self.input_dtype[0].split(',')) + required_parameters = set() + index_value = -1 + + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + required_parameter = [] + op_node = {} + + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + if self.is_set_for_bin_query() and self.input_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.input_dtype_for_bin_list[idx][i] + para['format'] = ifmts[i] + if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.input_fmt_for_bin_list[idx][i] + para['paramType'] = itype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + + input_parameter_key = (idtypes[i], ifmts[i]) + if itype == 'dynamic': + inputs.append([para]) + required_parameter.append(input_parameter_key) + elif itype == 'required': + inputs.append(para) + required_parameter.append(input_parameter_key) + else: + inputs.append(para) + + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + if self.is_set_for_bin_query() and self.output_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.output_dtype_for_bin_list[idx][i] + para['format'] = ofmts[i] + if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.output_fmt_for_bin_list[idx][i] + para['paramType'] = otype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + output_parameter_key = (odtypes[i], ofmts[i]) + if otype == 'dynamic': + outputs.append([para]) + required_parameter.append(output_parameter_key) + elif otype == 'required': + outputs.append(para) + required_parameter.append(output_parameter_key) + else: + outputs.append(para) + + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + + required_parameter_tuple = tuple(required_parameter) + if required_parameter_tuple in required_parameters: + continue + else: + required_parameters.add(required_parameter_tuple) + index_value +=1 + + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) + + + def _write_build_cmd(self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str): + hard_soc = const_var.conv_soc_ver(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + + bin_cmd_str = 'res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ + --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic ' + + build_cmd_var = "#!/bin/bash\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' + plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") + plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") + if plog_level is None: + build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR + if plog_stdout is None: + build_cmd_var += const_var.SET_PLOG_STDOUT + build_cmd_var += const_var.SRC_ENV + if hard_soc == "Ascend610Lite": + build_cmd_var += f'export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n' + build_cmd_var += bin_cmd_str.format(fun=self.op_intf, soc=hard_soc, param=param_file, + impl='high_performance,optional') + enable_tiling_keys = False + if self.tiling_keys: + tiling_keys_list = sorted(list(self.tiling_keys)) + tiling_key_str = ','.join([str(_key) for _key in tiling_keys_list]) + build_cmd_var += f' --tiling_key="{tiling_key_str}"' + enable_tiling_keys = True + + if self.op_debug_config: + op_debug_str = ','.join([str(_key) for _key in list(self.op_debug_config)]) + build_cmd_var += f' --op_debug_config={op_debug_str}' + + build_cmd_var += ")\n" + build_cmd_var += "\n" + if enable_tiling_keys is False: + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + else: + build_cmd_var += "if [ $? -eq 1 ]; then\n" + build_cmd_var += " if echo \"${res}\" | \ +grep -q \"None of the given tiling keys are in the supported list\"; then\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " else\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " exit 1\n" + build_cmd_var += " fi\n" + build_cmd_var += "else\n" + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + build_cmd_var += "fi\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' + + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write(build_cmd_var) + + +def get_tiling_keys(tiling_keys: str) -> Set: + all_tiling_keys = set() + if not tiling_keys: + return all_tiling_keys + + tiling_key_list = tiling_keys.split(';') + for tiling_key_value in tiling_key_list: + pattern = r"(? int(end): + continue + for i in range(int(start), int(end) + 1): + all_tiling_keys.add(i) + elif tiling_key_value.isdigit(): + all_tiling_keys.add(int(tiling_key_value)) + return all_tiling_keys + + +def trans_soc_verion(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: + return low_soc_ver + return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: + tiling_key_info = defaultdict(set) + op_debug_config = defaultdict(set) + if not opc_config_file: + return tiling_key_info, op_debug_config + + if not os.path.exists(opc_config_file): + return tiling_key_info, op_debug_config + + with open(opc_config_file, 'r') as file: + contents = file.readlines() + + for _content in contents: + content = _content.strip() + opc_configs = content.split('@') + if len(opc_configs) < 3: + continue + + op_type = opc_configs[0] + if not op_type: + continue + + compute_unit = opc_configs[1] + if compute_unit: + compute_unit_list = compute_unit.split(';') + soc_lists = [] + for soc_ver in compute_unit_list: + short_soc_ver = trans_soc_verion(soc_ver) + soc_lists.append(short_soc_ver) + if soc not in soc_lists: + continue + + for options in opc_configs[2:]: + if "--tiling_key" in options: + format_tiling_keys = get_tiling_keys(options.split('=')[1]) + if format_tiling_keys: + tiling_key_info[op_type].update(format_tiling_keys) + if "--op_debug_config" in options: + format_debug_config = set(options.split("=")[1].split(';')) + if format_debug_config: + op_debug_config[op_type].update(format_debug_config) + + return tiling_key_info, op_debug_config + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str, + opc_config_file: str = '', ops: list = None): + if not os.path.exists(cfgfile): + print(f'INFO: {cfgfile} does not exists in this project, skip generating compile commands.') + return + + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) + tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) + auto_gen_path_dir = os.path.dirname(cfgfile) + all_soc_key = "ALL" + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + if op_desc.op_type in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) + if all_soc_key in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[all_soc_key]) + if op_desc.op_type in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) + if all_soc_key in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[all_soc_key]) + op_desc.gen_input_json(auto_gen_path_dir) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opc-config-file', nargs='?', const='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + if len(args.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(args.argv[1], + args.argv[2], + args.argv[3], + opc_config_file=args.opc_config_file) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_gen_options.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_gen_options.py new file mode 100755 index 000000000..2dd8ee047 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_gen_options.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import sys +import stat +import os +import re +import json +import const_var + + +def write_options_to_file(file_name: str, options_str: str, \ + op_type: str, compute_unit: str, split_char: str): + flags = os.O_WRONLY | os.O_CREAT + modes = stat.S_IWUSR | stat.S_IRUSR + try: + with os.fdopen(os.open(file_name, flags, modes), 'a') as fd: + fd.write(op_type + split_char + compute_unit + split_char + options_str + '\n') + except Exception as err: + print("write compile options config file failed") + raise(err) + + +def gen_compile_options(compile_options_file: str, op_type: str, \ + compute_unit: str, compile_options: list): + base_dir = os.path.dirname(compile_options_file) + opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") + compile_opt = [] + opc_debug_config = [] + opc_tiling_keys = "" + for opts in compile_options: + if "oom" in opts: + if opts == "--oom": + opc_debug_config.append("oom") + else: + raise RuntimeError(f"Unknown oom option format {opts}") + elif "--save-temp-files" in opts: + opc_debug_config.append("dump_cce") + elif "--tiling_key" in opts: + keys = opts.strip().split('=')[1].split(',') + keys_str = ";".join([key for key in keys]) + opc_tiling_keys = keys_str + else: + compile_opt.append(opts) + if len(compile_opt) > 0: + options_str = ';'.join([opt for opt in compile_opt]) + write_options_to_file(compile_options_file, options_str, op_type, compute_unit, ",") + opc_config_str = "" + if opc_debug_config: + opc_config_str = "--op_debug_config=" + ';'.join([opt for opt in opc_debug_config]) + if len(opc_tiling_keys) > 0: + if opc_config_str != "": + opc_config_str += "@" + opc_config_str += "--tiling_key=" + opc_tiling_keys + + if opc_config_str != "": + write_options_to_file(opc_config_file, opc_config_str, op_type, compute_unit, "@") + + +if __name__ == '__main__': + if len(sys.argv) < 4: + raise RuntimeError('arguments must greater than 4') + compute_soc = "" + comp_options = [] + for i in range(len(sys.argv) - 3): + if sys.argv[i + 3].upper().startswith("ASCEND"): + compute_soc += sys.argv[i + 3] + ";" + else: + comp_options.append(sys.argv[i + 3]) + if compute_soc != "": + compute_soc = compute_soc[0:-1] + gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_impl_build.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..caa3ab149 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_impl_build.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" + +import argparse +import glob +import sys +import os +import re +import datetime +from typing import List + +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = '''#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. +""" + +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex32": ["DT_COMPLEX32", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) + raise RuntimeError("cannot open %s" %(so_path)) + else: + msg = "load so succ " + so_path + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options.extend(compile_option_list[shortsoc]) + if '__ALLSOC__' in compile_option_list: + compile_options.extend(compile_option_list['__ALLSOC__']) + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): +{} + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = os.environ.get('BISHENG_REAL_PATH') + if bisheng is None: + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_current_build_config("enable_deterministic_mode") == 1: + options.append("-DDETERMINISTIC_MODE=1") + else: + options.append("-DDETERMINISTIC_MODE=0") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + msg = "start replay Acend C Operator {}, kernel name is {}" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API_BUILT_IN = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + + op_compile_option = '{}' + opp_path = os.environ.get('ASCEND_OPP_PATH') + dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) + if opp_path and os.path.exists(dat_path): + # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op + abs_rel_kernel_src_path = "{}" + extend_options = {{}} + extend_options['opp_kernel_hidden_dat_path'] = dat_path + compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ + extend_options) + else: + raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code == 0: + sup = "True" + reason = "" + elif err_code == 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + snake_case += "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + snake_case += "_" + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all: list = None): + self._build_paradefault() + if os.environ.get('BUILD_BUILTIN_OPP') != '1' and impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + print(f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check.") + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd, impl_path) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self: any, op_compile_option_all: list = None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format('OPTION', att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_virtual(self: any) -> str: + virt_exp = [] + for index in range(len(self.input_name)): + if self.input_virt.get(index) is None: + continue + val = [] + val.append('"param_name":"{}"'.format(self.input_name[index])) + val.append('"index":{}'.format(index)) + val.append('"dtype":"{}"'.format(self.input_dtype[index].split(',')[0])) + val.append('"format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"paramType":"optional"') + val.append('"shape":[1]') + val.append('"ori_shape":[1]') + virt_exp.append(' ' + self.input_name[index] + ' = {' + ','.join(val) + '}') + if len(virt_exp) > 0: + return '\n'.join(virt_exp) + else: + return ' # do ascendc build step' + + def _build_mc2_ctx(self: any): + if len(self.mc2_ctx) != 0: + return '["' + '", "'.join(self.mc2_ctx) + '"]' + return '[]' + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + in_idx = 0 + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + in_idx += 1 + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + now = datetime.datetime.now() + curr_year = now.year + former_year = curr_year - 1 + fd.write(IMPL_HEAD.format(former_year, curr_year, self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): + src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + + def _write_impl(self: any, fd: object, impl_path: str = ""): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + virt_exprs = self._build_virtual() + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, virt_exprs, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file,\ + self.op_type, self.op_file, self.param_type_dynamic, self.op_compile_option)) + else: + if os.environ.get('BUILD_BUILTIN_OPP') == '1': + relative_kernel_src_path = os.path.realpath(self._get_kernel_source(impl_path, src,\ + optype_snake(self.op_type), optype_snake_ex(self.op_type))) + # to match src path in .dat file system, turn relative path into absolute path + abs_rel_kernel_src_path = os.path.join("/", os.path.relpath(relative_kernel_src_path, impl_path)) + + # compiling hidden src file requires src path before packaging .dat file, + # hard code such src path to .py + fd.write(COMPILE_OP_API_BUILT_IN.format(self.op_type, self.op_type,\ + ', '.join(self.input_name), ', '.join(self.output_name), self.param_type_dynamic,\ + self._build_mc2_ctx(), self.input_type + self.output_type, self.output_init_value,\ + self.output_shape_depend_on_compute, self.op_compile_option, abs_rel_kernel_src_path)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.param_type_dynamic, self._build_mc2_ctx(),\ + self.input_type + self.output_type, self.output_init_value, self.output_shape_depend_on_compute,\ + self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + + +class OpFileNotExistsError(Exception): + """File does not exist error.""" + def __str__(self) -> str: + return f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" + + +def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: + """Get all ops info files.""" + ops_info_files = [] + for _dir in opsinfo_dir: + ops_info_files.extend(glob.glob(f'{_dir}/aic-*-ops-info.ini')) + return sorted(ops_info_files) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opsinfo-dir', nargs='*', default=None) + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + + if len(args.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] + + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] + + ops_infos = [] + if args.opsinfo_dir: + ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) + if not ops_infos: + raise OpFileNotExistsError(args.opsinfo_dir) + else: + ops_infos.append(args.argv[1]) + + for ops_info in ops_infos: + write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_ops_config.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..8147f9139 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_ops_config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. +""" + +import os +import glob +import json +import sys +import argparse +import const_var + + +BINARY_INFO_CONFIG_JSON = "binary_info_config.json" + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return sorted(all_suffix_files) + + +def add_dict_key(dict_to_add, key, value): + if value is None: + return + dict_to_add[key] = value + + +def correct_format_mode(format_mode): + if format_mode == 'FormatDefault': + return 'nd_agnostic' + if format_mode == 'FormatAgnostic': + return 'static_nd_agnostic' + if format_mode == 'FormatFixed': + return 'normal' + return format_mode + + +def get_input_or_output_config(in_or_out): + param_dict = {} + name = in_or_out.get('name') + index = in_or_out.get('index') + param_type = in_or_out.get('paramType') + + format_match_mode = in_or_out.get('format_match_mode') + format_mode = correct_format_mode(format_match_mode) + + dtype_mode = in_or_out.get('dtype_match_mode') + if dtype_mode == 'DtypeByte': + dtype_mode = 'bit' + + add_dict_key(param_dict, 'name', name) + add_dict_key(param_dict, 'index', index) + add_dict_key(param_dict, 'paramType', param_type) + add_dict_key(param_dict, 'dtypeMode', dtype_mode) + add_dict_key(param_dict, 'formatMode', format_mode) + return param_dict + + +def get_inputs_or_outputs_config(inputs_or_outputs): + if inputs_or_outputs is None: + return None + inputs_or_outputs_list = [] + + for in_or_out in inputs_or_outputs: + if isinstance(in_or_out, dict): + dict_param_config = get_input_or_output_config(in_or_out) + inputs_or_outputs_list.append(dict_param_config) + elif isinstance(in_or_out, list): + param_info = in_or_out[0] + list_param_config = get_input_or_output_config(param_info) + tmp_list = [list_param_config] + inputs_or_outputs_list.append(tmp_list) + return inputs_or_outputs_list + + +def gen_attrs_config(attrs): + attrs_list = [] + for attr in attrs: + attrs_dict = {} + name = attr.get('name') + mode = attr.get('mode') + add_dict_key(attrs_dict, 'name', name) + add_dict_key(attrs_dict, 'mode', mode) + attrs_list.append(attrs_dict) + return attrs_list + + +def get_params_config(support_info): + params_dict = {} + + inputs = support_info.get('inputs') + inputs_list = get_inputs_or_outputs_config(inputs) + params_dict['inputs'] = inputs_list + + outputs = support_info.get('outputs') + outputs_list = get_inputs_or_outputs_config(outputs) + params_dict['outputs'] = outputs_list + + attrs = support_info.get('attrs') + if attrs is not None: + attrs_list = gen_attrs_config(attrs) + params_dict['attrs'] = attrs_list + + return params_dict + + +def add_simplified_config(op_type, support_info, core_type, task_ration, objfile, config): + simplified_key = support_info.get('simplifiedKey') + + json_path = objfile.split('.')[0] + '.json' + + simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {'dynamicRankSupport': True} + + simplified_key_mode = support_info.get('simplifiedKeyMode') + add_dict_key(op_cfg, 'simplifiedKeyMode', simplified_key_mode) + + optional_input_mode = support_info.get('optionalInputMode') + optional_output_mode = support_info.get('optionalOutputMode') + add_dict_key(op_cfg, 'optionalInputMode', optional_input_mode) + if optional_output_mode is not None: + add_dict_key(op_cfg, 'optionalOutputMode', optional_output_mode) + + params_info = get_params_config(support_info) + op_cfg['params'] = params_info + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'multiKernelType': 1, 'binPath': objfile, 'jsonPath': json_path}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'binPath': objfile, 'jsonPath': json_path}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {'binList': []} + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE': 3, 'MIX_VECTOR_CORE': 4, 'MIX_AIV': 4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = contents.get("coreType") + task_ration = contents.get("taskRation") + core_type = core_type_map.get(core_type, -1) + if core_type == -1 and soc != 'ascend310b': + raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") + + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + + add_dict_key(bin_info, 'implMode', support_info.get('implMode')) + add_dict_key(bin_info, 'int64Mode', support_info.get('int64Mode')) + add_dict_key(bin_info, 'simplifiedKeyMode', support_info.get('simplifiedKeyMode')) + + simplified_key = support_info.get('simplifiedKey') + if simplified_key is not None: + bin_info['simplifiedKey'] = simplified_key + obj_file = os.path.join(soc, op_dir, bin_file_name) + add_simplified_config(op_type, support_info, core_type, task_ration, obj_file, config) + + add_dict_key(bin_info, 'dynamicParamMode', support_info.get('dynamicParamMode')) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + + add_dict_key(bin_info, 'opMode', support_info.get('opMode')) + add_dict_key(bin_info, 'optionalInputMode', support_info.get('optionalInputMode')) + add_dict_key(bin_info, 'deterministic', support_info.get('deterministic')) + if support_info.get('optionalOutputMode') is not None: + add_dict_key(bin_info, 'optionalOutputMode', support_info.get('optionalOutputMode')) + + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def check_single_op_is_void(root_dir): + for root, dirs, _ in os.walk(root_dir): + for sub_dir in dirs: + dir_path = os.path.join(root, sub_dir) + if len(os.listdir(dir_path)) == 0: + print(f"[ERROR] op {sub_dir}: not any obj compile success") + sys.exit(1) + + +def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): + suffix = 'json' + config = {BINARY_INFO_CONFIG_JSON: {}} + check_single_op_is_void(root_dir) + all_json_files = get_specified_suffix_file(root_dir, suffix) + + for _json in all_json_files: + gen_ops_config(_json, soc, config) + file_path = soc + _json.split(soc)[1] + with open(_json, "r+") as f: + data = json.load(f) + data["filePath"] = file_path + f.seek(0) + json.dump(data, f, indent=" ") + f.truncate() + + for cfg_key in config.keys(): + if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: + continue + cfg_file = os.path.join(out_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +# Parse multiple soc_versions ops in single path. +def gen_all_soc_config(all_path): + soc_roots = glob.glob(os.path.join(all_path, "ascend*")) + + for soc_root in soc_roots: + soc = os.path.basename(soc_root) + gen_all_config(soc_root, soc, soc_root, True) + cfg_files = glob.glob(os.path.join(soc_root, "*.json")) + cfg_path = os.path.join(all_path, "config", soc) + os.makedirs(cfg_path, exist_ok=True) + for cfg_file in cfg_files: + new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) + os.rename(cfg_file, new_file) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + + parser.add_argument('-o', + '--out', + nargs='?', + help='Output directory.') + + parser.add_argument('--skip-binary-info-config', + action='store_true', + help='binary_info_config.json file is not parsed.') + + return parser.parse_args() + + +def main(): + args = args_prase() + if args.out is None: + out_dir = args.path + else: + out_dir = args.out + + gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_replay_build.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/batch_replay_impl.temp b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/code_channel_infer.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..3d1e4e87a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product == "ascend910_93" or op_product == "ascend910b": + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/const_var.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/const_var.py new file mode 100755 index 000000000..c7c41d9dd --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/const_var.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', + 'ascend910_93': 'Ascend910_9391', 'ascend610lite': 'Ascend610Lite'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" +SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" +SRC_ENV = ''' +while true; do + case "$1" in + --kernel-src=*) + export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) + shift + ;; + -*) + shift + ;; + *) + break + ;; + esac +done +''' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} + + +def conv_soc_ver(ver: str): + return SOC_MAP_EXT.get(ver) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..b49cf8b9b --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_ops_filter.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_version_info.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..5144edb1c --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/gen_version_info.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_op_info.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_simplified_keys.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..76f8ad286 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/insert_simplified_keys.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_entry.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_impl.temp b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/COPYING b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/README.md b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/VERSION b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/make-release.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself-header.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.1 b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.lsm b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/run-tests.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..8576d7616 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/opdesc_parser.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..fa7002b7a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/opdesc_parser.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910b4-1": "ascend910b", + "ascend910_9391": "ascend910_93", + "ascend910_9381": "ascend910_93", + "ascend910_9372": "ascend910_93", + "ascend910_9392": "ascend910_93", + "ascend910_9382": "ascend910_93", + "ascend910_9361": "ascend910_93", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a", + "ascend610lite": "ascend610lite" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_dtype_for_bin_list = [] + self.input_dtype_for_bin = {} + self.input_fmt = [] + self.input_fmt_for_bin_list = [] + self.input_fmt_for_bin = {} + self.input_virt = {} + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_dtype_for_bin_list = [] + self.output_dtype_for_bin = {} + self.output_fmt = [] + self.output_fmt_for_bin_list = [] + self.output_fmt_for_bin = {} + self.output_init_value = [] + self.output_shape_depend_on_compute = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + self.param_type_dynamic = False + self.mc2_ctx = [] + self.bin_cprs_list = [] + self.bin_cprs_head = [] + self.bin_save_list = [] + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + param_type = self._parse_str(conf) + self.input_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_dtype'.format(int(self.input_idx))): + self.input_dtype_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_format'.format(int(self.input_idx))): + self.input_fmt_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.virtual'.format(int(self.input_idx))): + self.input_virt[self.input_idx] = self._parse_str(conf) + elif conf.startswith('input{}.initValue'.format(int(self.input_idx))): + raise Exception(f'[ERROR]: Op: {{\'{self.op_type}\'}} input {self.input_ori_name[int(self.input_idx)]}\ + has InitValue, which is not support!') + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + self.output_init_value.append(None) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + param_type = self._parse_str(conf) + self.output_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_dtype'.format(int(self.output_idx))): + self.output_dtype_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_format'.format(int(self.output_idx))): + self.output_fmt_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.initValue'.format(int(self.output_idx))): + self.output_init_value[int(self.output_idx)] = self._parse_str(conf) + elif conf.startswith('output{}.outputShapeDependOnCompute=true'.format(int(self.output_idx))): + self.output_shape_depend_on_compute.append(int(self.output_idx)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'[ERROR]: The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} \ +conflicts with the built-in variable name. Use a complex name or prefix the operator name.') + + def parse_mc2_ctx(self: any, conf: str): + self.mc2_ctx = self._parse_list(conf) + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options.update(soc_ver_compile_options) + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + break + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('mc2.ctx'): + op_desc.parse_mc2_ctx(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/parse_ini_to_json.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..52dff4ad3 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/parse_ini_to_json.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights£¬group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + parse_ini_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if not ini_file_path_list: + ini_file_path_list.append("tbe_ops_info.ini") + + for ini_file in ini_file_path_list: + if os.path.exists(ini_file): + parse_ini_list.append(ini_file) + + if parse_ini_list: + if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/preset_parse.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/preset_parse.py new file mode 100755 index 000000000..4ff54c0a7 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/preset_parse.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +import json +import sys +import os + + +def read_json(file): + with open(file, 'r') as fd: + config = json.load(fd) + return config + + +def get_config_opts(file): + config = read_json(file) + + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_codegen.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_impl.temp b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/tiling_data_def_build.py b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..cb2a01c56 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/cmake/util/tiling_data_def_build.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#undef GET_TILING_DATA +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc new file mode 100644 index 000000000..6b6fc00c7 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("Demo") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Demo") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/CMakeLists.txt new file mode 100644 index 000000000..2b8d9034f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/CMakeLists.txt @@ -0,0 +1,93 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +file(GLOB group_proto_src ${ASCEND_AUTOGEN_PATH}/group_proto/*.cc) + +add_library(cust_op_proto SHARED + $<$:${group_proto_src}> + ${ops_srcs} + ${ASCEND_AUTOGEN_PATH}/op_proto.cc +) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +file(GLOB GROUP_PROTO_HEADERS ${ASCEND_AUTOGEN_PATH}/group_proto/*.h) +if (GROUP_PROTO_HEADERS) + install(FILES ${GROUP_PROTO_HEADERS} + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +endif() +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp new file mode 100644 index 000000000..ddcf8818c --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp @@ -0,0 +1,61 @@ + +#include "demo_tiling.h" +#include "register/op_def_registry.h" + + +namespace optiling { +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + + DemoTilingData tiling; + uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); + + tiling.set_size(inputNum); + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class Demo : public OpDef { +public: + explicit Demo(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend910b"); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(Demo); +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h new file mode 100644 index 000000000..9dfd12fa5 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -0,0 +1,10 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(DemoTilingData) + TILING_DATA_FIELD_DEF(uint32_t, size); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp new file mode 100644 index 000000000..2170ab50f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -0,0 +1,67 @@ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue +class KernelDemo { +public: + __aicore__ inline KernelDemo() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, uint32_t totalLength) { + this->totalLength = totalLength; + xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); + yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); + + } + __aicore__ inline void Process() { + CopyIn(); + Compute(); + CopyOut(); + } + +private: + // TODO: user kernel private member + + __aicore__ inline void CopyIn(){ + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::DataCopy(xLocal, xGm, this->totalLength); + inQueueX.EnQue(xLocal); + } + + __aicore__ inline void Compute(){ + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = outQueueY.AllocTensor(); + float x = 0; + float y = 0; + half sum = 0; + for (size_t i = 0; i < this->totalLength; i++) + { + x= static_cast(xLocal.GetValue(i)); + y= static_cast(sum); + x += y; + sum =static_cast(x); + yLocal.SetValue(i, sum); + } + + + outQueueY.EnQue(yLocal); + inQueueX.FreeTensor(xLocal); + } + __aicore__ inline void CopyOut(){ + AscendC::LocalTensor yLocal = outQueueY.DeQue(); + AscendC::DataCopy(yGm, yLocal, this->totalLength); + outQueueY.FreeTensor(yLocal); + } +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX; + AscendC::TQue outQueueY; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + uint32_t totalLength; +}; + +extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + KernelDemo op; + op.Init(x, y, tiling_data.size); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/install.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/install.sh new file mode 100755 index 000000000..b87a9790f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + exit 1 + fi +fi +chmod u+w ${targetdir}/vendors + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}\nexport LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + log "[ERROR] echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ export LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH} ] to set the environment path" +fi + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/upgrade.sh b/operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/upgrade.sh new file mode 100755 index 000000000..65457d2dd --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/scripts/upgrade.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + log "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakeLists.txt new file mode 100644 index 000000000..ea2c509e5 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakeLists.txt @@ -0,0 +1,74 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + if(CMAKE_CROSS_LIBRARY_PATH) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${CMAKE_CROSS_LIBRARY_PATH}) + endif() + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakePresets.json b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakePresets.json new file mode 100644 index 000000000..d32bf6d9a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b;ascend910b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/build.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/build.sh new file mode 100755 index 000000000..113fa60d8 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/build.sh @@ -0,0 +1,95 @@ +#!/bin/bash +if [ -z "$BASE_LIBS_PATH" ]; then + if [ -z "$ASCEND_HOME_PATH" ]; then + if [ -z "$ASCEND_AICPU_PATH" ]; then + echo "please set env." + exit 1 + else + export ASCEND_HOME_PATH=$ASCEND_AICPU_PATH + fi + else + export ASCEND_HOME_PATH=$ASCEND_HOME_PATH + fi +else + export ASCEND_HOME_PATH=$BASE_LIBS_PATH +fi +echo "using ASCEND_HOME_PATH: $ASCEND_HOME_PATH" +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + else + cmake .. --preset=default -DASCEND_CANN_PACKAGE_PATH=${ASCEND_HOME_PATH} + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/config.cmake b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/config.cmake new file mode 100755 index 000000000..72690b9e4 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/config.cmake @@ -0,0 +1,30 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_AUTOGEN_GROUPPROTO_PATH ${CMAKE_BINARY_DIR}/autogen/group_proto) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH} ${ASCEND_AUTOGEN_GROUPPROTO_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +set(CUSTOM_OPC_OPTIONS "custom_opc_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + ) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/func.cmake b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/func.cmake new file mode 100755 index 000000000..c4da5fee8 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/func.cmake @@ -0,0 +1,212 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX;ENABLE_SOURCE" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -I ${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + + set(ENV{ENABLE_SOURCE_PACAKGE} ${OPBUILD_ENABLE_SOURCE}) + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + unset(ENV{ENABLE_SOURCE_PACAKGE}) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_gen_options.py + ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} ${OP_TYPE} ${OP_COMPILE_COMPUTE_UNIT} + ${OP_COMPILE_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR) + if (${EXEC_RESULT}) + message("add ops compile options info: ${EXEC_INFO}") + message("add ops compile options error: ${EXEC_ERROR}") + message(FATAL_ERROR "add ops compile options failed!") + endif() +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + --opc-config-file ${ASCEND_AUTOGEN_PATH}/${CUSTOM_OPC_OPTIONS} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + # add Environment Variable Configurations of python & ccache + set(_ASCENDC_ENV_VAR) + list(APPEND _ASCENDC_ENV_VAR export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} &&) + # whether need judging CMAKE_C_COMPILER_LAUNCHER + if(${CMAKE_CXX_COMPILER_LAUNCHER} MATCHES "ccache$") + list(APPEND _ASCENDC_ENV_VAR export ASCENDC_CCACHE_EXECUTABLE=${CMAKE_CXX_COMPILER_LAUNCHER} &&) + endif() + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND ${_ASCENDC_ENV_VAR} bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/intf.cmake b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/intf.cmake new file mode 100755 index 000000000..d2643bbc7 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/intf.cmake @@ -0,0 +1,28 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include + ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel +) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/makeself.cmake b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/makeself.cmake new file mode 100755 index 000000000..2c482b042 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/makeself.cmake @@ -0,0 +1,33 @@ +execute_process(COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) + +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() + +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + +if (NOT "${CPACK_PACKAGE_DIRECTORY}x" STREQUAL "${CPACK_INSTALL_PREFIX}x") + execute_process( + COMMAND ${CMAKE_COMMAND} -E make_directory ${CPACK_INSTALL_PREFIX} + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) + + execute_process( + COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_INSTALL_PREFIX}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_INSTALL_PREFIX}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + ) +endif() + + diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/__init__.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/__init__.py new file mode 100755 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_bin_param_build.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_bin_param_build.py new file mode 100755 index 000000000..39bfbb13c --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import argparse +import sys +import os +import json +import hashlib +import re +import copy +from collections import defaultdict +from typing import Dict, List, Set, Tuple, NamedTuple + +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ParamInfo(NamedTuple): + dtype_list: list + format_list: list + dtype_for_bin_list: dict + format_for_bin_list: dict + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + self.tiling_keys = set() + self.op_debug_config = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def set_tiling_key(self: any, tiling_key_info: Set): + if tiling_key_info: + self.tiling_keys.update(tiling_key_info) + + def set_op_debug_config(self: any, op_debug_config: str): + if op_debug_config: + self.op_debug_config = op_debug_config + + def get_full_list(self: any): + dtype_list = [] + for dtype_in in self.input_dtype: + dtype_list.append(dtype_in.split(',')) + for dtype_out in self.output_dtype: + dtype_list.append(dtype_out.split(',')) + + format_list = [] + for fmt_in in self.input_fmt: + format_list.append(fmt_in.split(',')) + for fmt_out in self.output_fmt: + format_list.append(fmt_out.split(',')) + + dtype_for_bin_list = [[] for _ in range(len(self.input_dtype) + len(self.output_dtype))] + format_for_bin_list = copy.deepcopy(dtype_for_bin_list) + + for key, value in self.input_dtype_for_bin.items(): + dtype_for_bin_list[key] = value.split(',') + for key, value in self.output_dtype_for_bin.items(): + dtype_for_bin_list[key + len(self.input_dtype)] = value.split(',') + for key, value in self.input_fmt_for_bin.items(): + format_for_bin_list[key] = value.split(',') + for key, value in self.output_fmt_for_bin.items(): + format_for_bin_list[key + len(self.input_dtype)] = value.split(',') + + return ParamInfo(dtype_list, format_list, dtype_for_bin_list, format_for_bin_list) + + + def gen_bin_cprs_list(self: any, param_info: ParamInfo): + combine_dict = {} + origin_combine_dict = {} + for cob_idx in range(0, len(self.input_dtype[0].split(','))): + origin_combine = "" + combine = "" + for param_idx in range(0, len(self.input_dtype) + len(self.output_dtype)): + if (param_info.dtype_for_bin_list[param_idx]): + combine += param_info.dtype_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.dtype_list[param_idx][cob_idx] + origin_combine += param_info.dtype_list[param_idx][cob_idx] + if (param_info.format_for_bin_list[param_idx]): + combine += param_info.format_for_bin_list[param_idx][cob_idx] + else: + combine += param_info.format_list[param_idx][cob_idx] + origin_combine += param_info.format_list[param_idx][cob_idx] + if (combine not in combine_dict): + combine_dict[combine] = [] + combine_dict[combine].append(cob_idx) + origin_combine_dict[origin_combine] = cob_idx + for key, value in combine_dict.items(): + if (key not in origin_combine_dict): + print(f"WARNING: ForBinQuery {key} not in origin combine") + self.bin_save_list += value + continue + if len(value) == 1 and value[0] == origin_combine_dict[key]: + self.bin_save_list += value + continue + self.bin_cprs_head.append(origin_combine_dict[key]) + self.bin_cprs_list.append(value) + for index, sub_list in enumerate(self.bin_cprs_list): + if self.bin_cprs_head[index] not in self.bin_save_list: + continue + sub_list.append(self.bin_cprs_head[index]) + self.bin_save_list += self.bin_cprs_head + + + def gen_for_bin_list(self: any, param_info: ParamInfo): + combine_size = len(self.input_dtype[0].split(',')) + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + + self.input_dtype_for_bin_list = [[] for _ in range(input_size)] + self.output_dtype_for_bin_list = [[] for _ in range(output_size)] + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [[] for _ in range(combine_size)] + self.input_fmt_for_bin_list = copy.deepcopy(self.input_dtype_for_bin_list) + self.output_fmt_for_bin_list = copy.deepcopy(self.output_dtype_for_bin_list) + + for index, sub_list in enumerate(self.bin_cprs_list): + head_idx = self.bin_cprs_head[index] + for cmb_idx in sub_list: + for i in range(0, input_size): + self.input_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i][cmb_idx]) + self.input_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i][cmb_idx]) + for i in range(0, output_size): + self.output_dtype_for_bin_list[i][head_idx].append(param_info.dtype_list[i + input_size][cmb_idx]) + self.output_fmt_for_bin_list[i][head_idx].append(param_info.format_list[i + input_size][cmb_idx]) + + + def rm_cprs_cmb(self: any, dtype_list, format_list, input_size, output_size): + for i in range(0, input_size): + self.input_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.input_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.input_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.input_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.input_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.input_fmt[i] = new_format_str[:-1] + for i in range(0, output_size): + self.output_dtype_for_bin_list[i] = [ + element for index, element in enumerate(self.output_dtype_for_bin_list[i]) + if index in self.bin_save_list + ] + self.output_fmt_for_bin_list[i] = [ + element for index, element in enumerate(self.output_fmt_for_bin_list[i]) + if index in self.bin_save_list + ] + new_dtype_list = [ + element for index, element in enumerate(dtype_list[i + input_size]) + if index in self.bin_save_list + ] + new_dtype_str = "" + for dtype in new_dtype_list: + new_dtype_str += f"{dtype}," + self.output_dtype[i] = new_dtype_str[:-1] + new_format_list = [ + element for index, element in enumerate(format_list[i + input_size]) + if index in self.bin_save_list + ] + new_format_str = "" + for fmt in new_format_list: + new_format_str += f"{fmt}," + self.output_fmt[i] = new_format_str[:-1] + + + def is_set_for_bin_query(self: any): + return any([ + self.input_dtype_for_bin, + self.output_dtype_for_bin, + self.input_fmt_for_bin, + self.output_fmt_for_bin, + ]) + + + def for_bin_list_match(self: any): + if not self.is_set_for_bin_query(): + return + input_size = len(self.input_dtype) + output_size = len(self.output_dtype) + param_info = self.get_full_list() + self.gen_bin_cprs_list(param_info) + self.gen_for_bin_list(param_info) + if len(self.bin_save_list) == len(self.input_dtype[0].split(',')): + print(f'WARNING: ForBinQuery can not compress number of bin file with this set, please check!!.') + return + self.rm_cprs_cmb(param_info.dtype_list, param_info.format_list, input_size, output_size) + + + def gen_input_json(self: any, auto_gen_path: str): + key_map = {} + self.for_bin_list_match() + count = len(self.input_dtype[0].split(',')) + required_parameters = set() + index_value = -1 + + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + required_parameter = [] + op_node = {} + + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + if self.is_set_for_bin_query() and self.input_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.input_dtype_for_bin_list[idx][i] + para['format'] = ifmts[i] + if self.is_set_for_bin_query() and self.input_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.input_fmt_for_bin_list[idx][i] + para['paramType'] = itype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + + input_parameter_key = (idtypes[i], ifmts[i]) + if itype == 'dynamic': + inputs.append([para]) + required_parameter.append(input_parameter_key) + elif itype == 'required': + inputs.append(para) + required_parameter.append(input_parameter_key) + else: + inputs.append(para) + + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + if self.is_set_for_bin_query() and self.output_dtype_for_bin_list[idx][i]: + para['dtypeForBinQuery'] = self.output_dtype_for_bin_list[idx][i] + para['format'] = ofmts[i] + if self.is_set_for_bin_query() and self.output_fmt_for_bin_list[idx][i]: + para['formatForBinQuery'] = self.output_fmt_for_bin_list[idx][i] + para['paramType'] = otype + para['shape'] = [-2] + para['format_match_mode'] = 'FormatAgnostic' + output_parameter_key = (odtypes[i], ofmts[i]) + if otype == 'dynamic': + outputs.append([para]) + required_parameter.append(output_parameter_key) + elif otype == 'required': + outputs.append(para) + required_parameter.append(output_parameter_key) + else: + outputs.append(para) + + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + + required_parameter_tuple = tuple(required_parameter) + if required_parameter_tuple in required_parameters: + continue + else: + required_parameters.add(required_parameter_tuple) + index_value +=1 + + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_build_cmd(param_file, bin_file, index_value, auto_gen_path) + + + def _write_build_cmd(self: any, param_file: str, bin_file: str, index: int, auto_gen_path: str): + hard_soc = const_var.conv_soc_ver(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + + bin_cmd_str = 'res=$(opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ + --output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic ' + + build_cmd_var = "#!/bin/bash\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} ..."\n' + plog_level = os.environ.get("ASCEND_GLOBAL_LOG_LEVEL") + plog_stdout = os.environ.get("ASCEND_SLOG_PRINT_TO_STDOUT") + if plog_level is None: + build_cmd_var += const_var.SET_PLOG_LEVEL_ERROR + if plog_stdout is None: + build_cmd_var += const_var.SET_PLOG_STDOUT + build_cmd_var += const_var.SRC_ENV + if hard_soc == "Ascend610Lite": + build_cmd_var += f'export ASCEND_CUSTOM_OPP_PATH={auto_gen_path}:$ASCEND_CUSTOM_OPP_PATH \n' + build_cmd_var += bin_cmd_str.format(fun=self.op_intf, soc=hard_soc, param=param_file, + impl='high_performance,optional') + enable_tiling_keys = False + if self.tiling_keys: + tiling_keys_list = sorted(list(self.tiling_keys)) + tiling_key_str = ','.join([str(_key) for _key in tiling_keys_list]) + build_cmd_var += f' --tiling_key="{tiling_key_str}"' + enable_tiling_keys = True + + if self.op_debug_config: + op_debug_str = ','.join([str(_key) for _key in list(self.op_debug_config)]) + build_cmd_var += f' --op_debug_config={op_debug_str}' + + build_cmd_var += ")\n" + build_cmd_var += "\n" + if enable_tiling_keys is False: + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + else: + build_cmd_var += "if [ $? -eq 1 ]; then\n" + build_cmd_var += " if echo \"${res}\" | \ +grep -q \"None of the given tiling keys are in the supported list\"; then\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " else\n" + build_cmd_var += " echo \"${res}\"\n" + build_cmd_var += " exit 1\n" + build_cmd_var += " fi\n" + build_cmd_var += "else\n" + build_cmd_var += "echo \"${res}\"\n" + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.json') + build_cmd_var += const_var.CHK_CMD.format(res_file=bin_file + '.o') + build_cmd_var += "fi\n" + build_cmd_var += f'echo "[{self.soc}] Generating {bin_file} Done"\n' + + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write(build_cmd_var) + + +def get_tiling_keys(tiling_keys: str) -> Set: + all_tiling_keys = set() + if not tiling_keys: + return all_tiling_keys + + tiling_key_list = tiling_keys.split(';') + for tiling_key_value in tiling_key_list: + pattern = r"(? int(end): + continue + for i in range(int(start), int(end) + 1): + all_tiling_keys.add(i) + elif tiling_key_value.isdigit(): + all_tiling_keys.add(int(tiling_key_value)) + return all_tiling_keys + + +def trans_soc_verion(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in opdesc_parser.SOC_TO_SHORT_SOC_MAP: + return low_soc_ver + return opdesc_parser.SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def parse_op_debug_confg(opc_config_file: str, soc: str) -> Dict: + tiling_key_info = defaultdict(set) + op_debug_config = defaultdict(set) + if not opc_config_file: + return tiling_key_info, op_debug_config + + if not os.path.exists(opc_config_file): + return tiling_key_info, op_debug_config + + with open(opc_config_file, 'r') as file: + contents = file.readlines() + + for _content in contents: + content = _content.strip() + opc_configs = content.split('@') + if len(opc_configs) < 3: + continue + + op_type = opc_configs[0] + if not op_type: + continue + + compute_unit = opc_configs[1] + if compute_unit: + compute_unit_list = compute_unit.split(';') + soc_lists = [] + for soc_ver in compute_unit_list: + short_soc_ver = trans_soc_verion(soc_ver) + soc_lists.append(short_soc_ver) + if soc not in soc_lists: + continue + + for options in opc_configs[2:]: + if "--tiling_key" in options: + format_tiling_keys = get_tiling_keys(options.split('=')[1]) + if format_tiling_keys: + tiling_key_info[op_type].update(format_tiling_keys) + if "--op_debug_config" in options: + format_debug_config = set(options.split("=")[1].split(';')) + if format_debug_config: + op_debug_config[op_type].update(format_debug_config) + + return tiling_key_info, op_debug_config + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str, + opc_config_file: str = '', ops: list = None): + if not os.path.exists(cfgfile): + print(f'INFO: {cfgfile} does not exists in this project, skip generating compile commands.') + return + + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, ops) + tiling_key_info, op_debug_config = parse_op_debug_confg(opc_config_file, soc) + auto_gen_path_dir = os.path.dirname(cfgfile) + all_soc_key = "ALL" + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + if op_desc.op_type in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[op_desc.op_type]) + if all_soc_key in op_debug_config: + op_desc.set_op_debug_config(op_debug_config[all_soc_key]) + if op_desc.op_type in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[op_desc.op_type]) + if all_soc_key in tiling_key_info: + op_desc.set_tiling_key(tiling_key_info[all_soc_key]) + op_desc.gen_input_json(auto_gen_path_dir) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opc-config-file', nargs='?', const='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + if len(args.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(args.argv[1], + args.argv[2], + args.argv[3], + opc_config_file=args.opc_config_file) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_gen_options.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_gen_options.py new file mode 100755 index 000000000..2dd8ee047 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_gen_options.py @@ -0,0 +1,86 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import sys +import stat +import os +import re +import json +import const_var + + +def write_options_to_file(file_name: str, options_str: str, \ + op_type: str, compute_unit: str, split_char: str): + flags = os.O_WRONLY | os.O_CREAT + modes = stat.S_IWUSR | stat.S_IRUSR + try: + with os.fdopen(os.open(file_name, flags, modes), 'a') as fd: + fd.write(op_type + split_char + compute_unit + split_char + options_str + '\n') + except Exception as err: + print("write compile options config file failed") + raise(err) + + +def gen_compile_options(compile_options_file: str, op_type: str, \ + compute_unit: str, compile_options: list): + base_dir = os.path.dirname(compile_options_file) + opc_config_file = os.path.join(base_dir, "custom_opc_options.ini") + compile_opt = [] + opc_debug_config = [] + opc_tiling_keys = "" + for opts in compile_options: + if "oom" in opts: + if opts == "--oom": + opc_debug_config.append("oom") + else: + raise RuntimeError(f"Unknown oom option format {opts}") + elif "--save-temp-files" in opts: + opc_debug_config.append("dump_cce") + elif "--tiling_key" in opts: + keys = opts.strip().split('=')[1].split(',') + keys_str = ";".join([key for key in keys]) + opc_tiling_keys = keys_str + else: + compile_opt.append(opts) + if len(compile_opt) > 0: + options_str = ';'.join([opt for opt in compile_opt]) + write_options_to_file(compile_options_file, options_str, op_type, compute_unit, ",") + opc_config_str = "" + if opc_debug_config: + opc_config_str = "--op_debug_config=" + ';'.join([opt for opt in opc_debug_config]) + if len(opc_tiling_keys) > 0: + if opc_config_str != "": + opc_config_str += "@" + opc_config_str += "--tiling_key=" + opc_tiling_keys + + if opc_config_str != "": + write_options_to_file(opc_config_file, opc_config_str, op_type, compute_unit, "@") + + +if __name__ == '__main__': + if len(sys.argv) < 4: + raise RuntimeError('arguments must greater than 4') + compute_soc = "" + comp_options = [] + for i in range(len(sys.argv) - 3): + if sys.argv[i + 3].upper().startswith("ASCEND"): + compute_soc += sys.argv[i + 3] + ";" + else: + comp_options.append(sys.argv[i + 3]) + if compute_soc != "": + compute_soc = compute_soc[0:-1] + gen_compile_options(sys.argv[1], sys.argv[2], compute_soc, comp_options) \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_impl_build.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_impl_build.py new file mode 100755 index 000000000..caa3ab149 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_impl_build.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" + +import argparse +import glob +import sys +import os +import re +import datetime +from typing import List + +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = '''#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Copyright (c) Huawei Technologies Co., Ltd. {}-{}. All rights reserved. +""" + +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.tikcpp.compile_op import CommonUtility, AscendCLogLevel +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex32": ["DT_COMPLEX32", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + CommonUtility.print_compile_log("", error, AscendCLogLevel.LOG_ERROR) + raise RuntimeError("cannot open %s" %(so_path)) + else: + msg = "load so succ " + so_path + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options.extend(compile_option_list[shortsoc]) + if '__ALLSOC__' in compile_option_list: + compile_options.extend(compile_option_list['__ALLSOC__']) + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, "..", "ascendc", os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): +{} + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = os.environ.get('BISHENG_REAL_PATH') + if bisheng is None: + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "..", "..", "include")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_current_build_config("enable_deterministic_mode") == 1: + options.append("-DDETERMINISTIC_MODE=1") + else: + options.append("-DDETERMINISTIC_MODE=0") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + msg = "start replay Acend C Operator {}, kernel name is {}" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + msg = "start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so" + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, param_type_dynamic = {}) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API_BUILT_IN = ''' + msg = "start compile Acend C Operator {}, kernel name is " + kernel_name + CommonUtility.print_compile_log("", msg, AscendCLogLevel.LOG_INFO) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}],\\ + param_type_dynamic = {}, mc2_ctx = {}, param_type_list = {}, init_value_list = {},\\ + output_shape_depend_on_compute = {}) + + op_compile_option = '{}' + opp_path = os.environ.get('ASCEND_OPP_PATH') + dat_path = os.path.realpath(os.path.join(opp_path, "built-in", "op_impl", "ai_core", "tbe", "ascendc_impl.dat")) + if opp_path and os.path.exists(dat_path): + # dat file exists: built in hidden src file online compiling process. append vfs compile option in compile_op + abs_rel_kernel_src_path = "{}" + extend_options = {{}} + extend_options['opp_kernel_hidden_dat_path'] = dat_path + compile_op(abs_rel_kernel_src_path, origin_func_name, op_info, options, code_channel, op_compile_option,\\ + extend_options) + else: + raise RuntimeError("built-in opp compile, ascendc_impl.dat file path does not exist: %s" %(dat_path)) +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code == 0: + sup = "True" + reason = "" + elif err_code == 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + snake_case += "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + snake_case += "_" + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all: list = None): + self._build_paradefault() + if os.environ.get('BUILD_BUILTIN_OPP') != '1' and impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + print(f"[ERROR]: operator: {self.op_file} source file: {src_file} does not found, please check.") + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd, impl_path) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self: any, op_compile_option_all: list = None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format('OPTION', att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_virtual(self: any) -> str: + virt_exp = [] + for index in range(len(self.input_name)): + if self.input_virt.get(index) is None: + continue + val = [] + val.append('"param_name":"{}"'.format(self.input_name[index])) + val.append('"index":{}'.format(index)) + val.append('"dtype":"{}"'.format(self.input_dtype[index].split(',')[0])) + val.append('"format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"ori_format":"{}"'.format(self.input_fmt[index].split(',')[0])) + val.append('"paramType":"optional"') + val.append('"shape":[1]') + val.append('"ori_shape":[1]') + virt_exp.append(' ' + self.input_name[index] + ' = {' + ','.join(val) + '}') + if len(virt_exp) > 0: + return '\n'.join(virt_exp) + else: + return ' # do ascendc build step' + + def _build_mc2_ctx(self: any): + if len(self.mc2_ctx) != 0: + return '["' + '", "'.join(self.mc2_ctx) + '"]' + return '[]' + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + in_idx = 0 + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + in_idx += 1 + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + now = datetime.datetime.now() + curr_year = now.year + former_year = curr_year - 1 + fd.write(IMPL_HEAD.format(former_year, curr_year, self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _get_kernel_source(self: any, kernel_src_dir, src_file, dir_snake, dir_ex): + src_ex = os.path.join(kernel_src_dir, dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.environ.get('BUILD_KERNEL_SRC') + if src and os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, src_file) + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_snake, dir_snake + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, dir_ex, dir_ex + ".cpp") + if os.path.exists(src): + return src + src = os.path.join(kernel_src_dir, os.path.splitext(src_file)[0], src_file) + if os.path.exists(src): + return src + return src_ex + + def _write_impl(self: any, fd: object, impl_path: str = ""): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + virt_exprs = self._build_virtual() + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, virt_exprs, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file,\ + self.op_type, self.op_file, self.param_type_dynamic, self.op_compile_option)) + else: + if os.environ.get('BUILD_BUILTIN_OPP') == '1': + relative_kernel_src_path = os.path.realpath(self._get_kernel_source(impl_path, src,\ + optype_snake(self.op_type), optype_snake_ex(self.op_type))) + # to match src path in .dat file system, turn relative path into absolute path + abs_rel_kernel_src_path = os.path.join("/", os.path.relpath(relative_kernel_src_path, impl_path)) + + # compiling hidden src file requires src path before packaging .dat file, + # hard code such src path to .py + fd.write(COMPILE_OP_API_BUILT_IN.format(self.op_type, self.op_type,\ + ', '.join(self.input_name), ', '.join(self.output_name), self.param_type_dynamic,\ + self._build_mc2_ctx(), self.input_type + self.output_type, self.output_init_value,\ + self.output_shape_depend_on_compute, self.op_compile_option, abs_rel_kernel_src_path)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), self.param_type_dynamic, self._build_mc2_ctx(),\ + self.input_type + self.output_type, self.output_init_value, self.output_shape_depend_on_compute,\ + self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + + +class OpFileNotExistsError(Exception): + """File does not exist error.""" + def __str__(self) -> str: + return f"File aic-*-ops-info.ini does not exist in directory {super().__str__()}" + + +def get_ops_info_files(opsinfo_dir: List[str]) -> List[str]: + """Get all ops info files.""" + ops_info_files = [] + for _dir in opsinfo_dir: + ops_info_files.extend(glob.glob(f'{_dir}/aic-*-ops-info.ini')) + return sorted(ops_info_files) + + +def parse_args(argv): + """Command line parameter parsing""" + parser = argparse.ArgumentParser() + parser.add_argument('argv', nargs='+') + parser.add_argument('--opsinfo-dir', nargs='*', default=None) + return parser.parse_args(argv) + + +if __name__ == '__main__': + args = parse_args(sys.argv) + + if len(args.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = args.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = args.argv[3] + + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = args.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = args.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = args.argv[6] + + ops_infos = [] + if args.opsinfo_dir: + ops_infos.extend(get_ops_info_files(args.opsinfo_dir)) + if not ops_infos: + raise OpFileNotExistsError(args.opsinfo_dir) + else: + ops_infos.append(args.argv[1]) + + for ops_info in ops_infos: + write_scripts(cfgfile=ops_info, cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_ops_config.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_ops_config.py new file mode 100755 index 000000000..8147f9139 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_ops_config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. +""" + +import os +import glob +import json +import sys +import argparse +import const_var + + +BINARY_INFO_CONFIG_JSON = "binary_info_config.json" + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return sorted(all_suffix_files) + + +def add_dict_key(dict_to_add, key, value): + if value is None: + return + dict_to_add[key] = value + + +def correct_format_mode(format_mode): + if format_mode == 'FormatDefault': + return 'nd_agnostic' + if format_mode == 'FormatAgnostic': + return 'static_nd_agnostic' + if format_mode == 'FormatFixed': + return 'normal' + return format_mode + + +def get_input_or_output_config(in_or_out): + param_dict = {} + name = in_or_out.get('name') + index = in_or_out.get('index') + param_type = in_or_out.get('paramType') + + format_match_mode = in_or_out.get('format_match_mode') + format_mode = correct_format_mode(format_match_mode) + + dtype_mode = in_or_out.get('dtype_match_mode') + if dtype_mode == 'DtypeByte': + dtype_mode = 'bit' + + add_dict_key(param_dict, 'name', name) + add_dict_key(param_dict, 'index', index) + add_dict_key(param_dict, 'paramType', param_type) + add_dict_key(param_dict, 'dtypeMode', dtype_mode) + add_dict_key(param_dict, 'formatMode', format_mode) + return param_dict + + +def get_inputs_or_outputs_config(inputs_or_outputs): + if inputs_or_outputs is None: + return None + inputs_or_outputs_list = [] + + for in_or_out in inputs_or_outputs: + if isinstance(in_or_out, dict): + dict_param_config = get_input_or_output_config(in_or_out) + inputs_or_outputs_list.append(dict_param_config) + elif isinstance(in_or_out, list): + param_info = in_or_out[0] + list_param_config = get_input_or_output_config(param_info) + tmp_list = [list_param_config] + inputs_or_outputs_list.append(tmp_list) + return inputs_or_outputs_list + + +def gen_attrs_config(attrs): + attrs_list = [] + for attr in attrs: + attrs_dict = {} + name = attr.get('name') + mode = attr.get('mode') + add_dict_key(attrs_dict, 'name', name) + add_dict_key(attrs_dict, 'mode', mode) + attrs_list.append(attrs_dict) + return attrs_list + + +def get_params_config(support_info): + params_dict = {} + + inputs = support_info.get('inputs') + inputs_list = get_inputs_or_outputs_config(inputs) + params_dict['inputs'] = inputs_list + + outputs = support_info.get('outputs') + outputs_list = get_inputs_or_outputs_config(outputs) + params_dict['outputs'] = outputs_list + + attrs = support_info.get('attrs') + if attrs is not None: + attrs_list = gen_attrs_config(attrs) + params_dict['attrs'] = attrs_list + + return params_dict + + +def add_simplified_config(op_type, support_info, core_type, task_ration, objfile, config): + simplified_key = support_info.get('simplifiedKey') + + json_path = objfile.split('.')[0] + '.json' + + simple_cfg = config.get(BINARY_INFO_CONFIG_JSON) + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {'dynamicRankSupport': True} + + simplified_key_mode = support_info.get('simplifiedKeyMode') + add_dict_key(op_cfg, 'simplifiedKeyMode', simplified_key_mode) + + optional_input_mode = support_info.get('optionalInputMode') + optional_output_mode = support_info.get('optionalOutputMode') + add_dict_key(op_cfg, 'optionalInputMode', optional_input_mode) + if optional_output_mode is not None: + add_dict_key(op_cfg, 'optionalOutputMode', optional_output_mode) + + params_info = get_params_config(support_info) + op_cfg['params'] = params_info + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'multiKernelType': 1, 'binPath': objfile, 'jsonPath': json_path}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': simplified_key, + 'binPath': objfile, 'jsonPath': json_path}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {'binList': []} + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE': 3, 'MIX_VECTOR_CORE': 4, 'MIX_AIV': 4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + core_type = contents.get("coreType") + task_ration = contents.get("taskRation") + core_type = core_type_map.get(core_type, -1) + if core_type == -1 and soc != 'ascend310b': + raise Exception("[ERROR]: must set coreType in json when soc version is {soc}.") + + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + + add_dict_key(bin_info, 'implMode', support_info.get('implMode')) + add_dict_key(bin_info, 'int64Mode', support_info.get('int64Mode')) + add_dict_key(bin_info, 'simplifiedKeyMode', support_info.get('simplifiedKeyMode')) + + simplified_key = support_info.get('simplifiedKey') + if simplified_key is not None: + bin_info['simplifiedKey'] = simplified_key + obj_file = os.path.join(soc, op_dir, bin_file_name) + add_simplified_config(op_type, support_info, core_type, task_ration, obj_file, config) + + add_dict_key(bin_info, 'dynamicParamMode', support_info.get('dynamicParamMode')) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + + add_dict_key(bin_info, 'opMode', support_info.get('opMode')) + add_dict_key(bin_info, 'optionalInputMode', support_info.get('optionalInputMode')) + add_dict_key(bin_info, 'deterministic', support_info.get('deterministic')) + if support_info.get('optionalOutputMode') is not None: + add_dict_key(bin_info, 'optionalOutputMode', support_info.get('optionalOutputMode')) + + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def check_single_op_is_void(root_dir): + for root, dirs, _ in os.walk(root_dir): + for sub_dir in dirs: + dir_path = os.path.join(root, sub_dir) + if len(os.listdir(dir_path)) == 0: + print(f"[ERROR] op {sub_dir}: not any obj compile success") + sys.exit(1) + + +def gen_all_config(root_dir, soc, out_dir, skip_binary_info_config): + suffix = 'json' + config = {BINARY_INFO_CONFIG_JSON: {}} + check_single_op_is_void(root_dir) + all_json_files = get_specified_suffix_file(root_dir, suffix) + + for _json in all_json_files: + gen_ops_config(_json, soc, config) + file_path = soc + _json.split(soc)[1] + with open(_json, "r+") as f: + data = json.load(f) + data["filePath"] = file_path + f.seek(0) + json.dump(data, f, indent=" ") + f.truncate() + + for cfg_key in config.keys(): + if skip_binary_info_config and cfg_key == BINARY_INFO_CONFIG_JSON: + continue + cfg_file = os.path.join(out_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +# Parse multiple soc_versions ops in single path. +def gen_all_soc_config(all_path): + soc_roots = glob.glob(os.path.join(all_path, "ascend*")) + + for soc_root in soc_roots: + soc = os.path.basename(soc_root) + gen_all_config(soc_root, soc, soc_root, True) + cfg_files = glob.glob(os.path.join(soc_root, "*.json")) + cfg_path = os.path.join(all_path, "config", soc) + os.makedirs(cfg_path, exist_ok=True) + for cfg_file in cfg_files: + new_file = os.path.join(cfg_path, os.path.basename(cfg_file)) + os.rename(cfg_file, new_file) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + + parser.add_argument('-o', + '--out', + nargs='?', + help='Output directory.') + + parser.add_argument('--skip-binary-info-config', + action='store_true', + help='binary_info_config.json file is not parsed.') + + return parser.parse_args() + + +def main(): + args = args_prase() + if args.out is None: + out_dir = args.path + else: + out_dir = args.out + + gen_all_config(args.path, args.soc, out_dir, args.skip_binary_info_config) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_replay_build.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_replay_build.py new file mode 100755 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/batch_replay_impl.temp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/code_channel_infer.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/code_channel_infer.py new file mode 100755 index 000000000..3d1e4e87a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product == "ascend910_93" or op_product == "ascend910b": + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/const_var.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/const_var.py new file mode 100755 index 000000000..c7c41d9dd --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/const_var.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', + 'ascend910_93': 'Ascend910_9391', 'ascend610lite': 'Ascend610Lite'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +SET_PLOG_LEVEL_ERROR = "export ASCEND_GLOBAL_LOG_LEVEL=3\n" +SET_PLOG_STDOUT = "export ASCEND_SLOG_PRINT_TO_STDOUT=1\n" +SRC_ENV = ''' +while true; do + case "$1" in + --kernel-src=*) + export BUILD_KERNEL_SRC=$(echo "$1" | cut -d"=" -f2-) + shift + ;; + -*) + shift + ;; + *) + break + ;; + esac +done +''' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} + + +def conv_soc_ver(ver: str): + return SOC_MAP_EXT.get(ver) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_impl_and_mrege_json.sh new file mode 100755 index 000000000..b49cf8b9b --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_ops_filter.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_ops_filter.sh new file mode 100755 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_version_info.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_version_info.sh new file mode 100755 index 000000000..5144edb1c --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/gen_version_info.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + + +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_op_info.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_op_info.py new file mode 100755 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_simplified_keys.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_simplified_keys.py new file mode 100755 index 000000000..76f8ad286 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/insert_simplified_keys.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import glob +import json +import argparse +import const_var + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'string': 13, + 'dual': 14, + 'dual': 15, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_entry.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_entry.py new file mode 100755 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_impl.temp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/COPYING b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/README.md b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/VERSION b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/make-release.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/make-release.sh new file mode 100755 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself-header.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself-header.sh new file mode 100755 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.1 b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.1 new file mode 100755 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.lsm b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.sh new file mode 100755 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/run-tests.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/run-tests.sh new file mode 100755 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/merge_aicpu_info_json.sh new file mode 100755 index 000000000..8576d7616 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/opdesc_parser.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/opdesc_parser.py new file mode 100755 index 000000000..fa7002b7a --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/opdesc_parser.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910b4-1": "ascend910b", + "ascend910_9391": "ascend910_93", + "ascend910_9381": "ascend910_93", + "ascend910_9372": "ascend910_93", + "ascend910_9392": "ascend910_93", + "ascend910_9382": "ascend910_93", + "ascend910_9361": "ascend910_93", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a", + "ascend610lite": "ascend610lite" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_dtype_for_bin_list = [] + self.input_dtype_for_bin = {} + self.input_fmt = [] + self.input_fmt_for_bin_list = [] + self.input_fmt_for_bin = {} + self.input_virt = {} + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_dtype_for_bin_list = [] + self.output_dtype_for_bin = {} + self.output_fmt = [] + self.output_fmt_for_bin_list = [] + self.output_fmt_for_bin = {} + self.output_init_value = [] + self.output_shape_depend_on_compute = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + self.param_type_dynamic = False + self.mc2_ctx = [] + self.bin_cprs_list = [] + self.bin_cprs_head = [] + self.bin_save_list = [] + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + param_type = self._parse_str(conf) + self.input_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_dtype'.format(int(self.input_idx))): + self.input_dtype_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + elif conf.startswith('input{}.for_bin_format'.format(int(self.input_idx))): + self.input_fmt_for_bin.update({self.input_idx : self._parse_str(conf)}) + elif conf.startswith('input{}.virtual'.format(int(self.input_idx))): + self.input_virt[self.input_idx] = self._parse_str(conf) + elif conf.startswith('input{}.initValue'.format(int(self.input_idx))): + raise Exception(f'[ERROR]: Op: {{\'{self.op_type}\'}} input {self.input_ori_name[int(self.input_idx)]}\ + has InitValue, which is not support!') + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + self.output_init_value.append(None) + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + param_type = self._parse_str(conf) + self.output_type.append(param_type) + if param_type == "dynamic": + self.param_type_dynamic = True + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_dtype'.format(int(self.output_idx))): + self.output_dtype_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + elif conf.startswith('output{}.for_bin_format'.format(int(self.output_idx))): + self.output_fmt_for_bin.update({self.output_idx : self._parse_str(conf)}) + elif conf.startswith('output{}.initValue'.format(int(self.output_idx))): + self.output_init_value[int(self.output_idx)] = self._parse_str(conf) + elif conf.startswith('output{}.outputShapeDependOnCompute=true'.format(int(self.output_idx))): + self.output_shape_depend_on_compute.append(int(self.output_idx)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'[ERROR]: The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} \ +conflicts with the built-in variable name. Use a complex name or prefix the operator name.') + + def parse_mc2_ctx(self: any, conf: str): + self.mc2_ctx = self._parse_list(conf) + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options.update(soc_ver_compile_options) + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return {} + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return {} + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + break + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('mc2.ctx'): + op_desc.parse_mc2_ctx(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/parse_ini_to_json.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/parse_ini_to_json.py new file mode 100755 index 000000000..52dff4ad3 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/parse_ini_to_json.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights£¬group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + parse_ini_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if not ini_file_path_list: + ini_file_path_list.append("tbe_ops_info.ini") + + for ini_file in ini_file_path_list: + if os.path.exists(ini_file): + parse_ini_list.append(ini_file) + + if parse_ini_list: + if not parse_ini_to_json(parse_ini_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/preset_parse.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/preset_parse.py new file mode 100755 index 000000000..4ff54c0a7 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/preset_parse.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +import json +import sys +import os + + +def read_json(file): + with open(file, 'r') as fd: + config = json.load(fd) + return config + + +def get_config_opts(file): + config = read_json(file) + + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_codegen.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_codegen.py new file mode 100755 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_impl.temp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/tiling_data_def_build.py b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/tiling_data_def_build.py new file mode 100755 index 000000000..cb2a01c56 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/cmake/util/tiling_data_def_build.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#undef GET_TILING_DATA +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc new file mode 100644 index 000000000..6b6fc00c7 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("Demo") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Demo") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/CMakeLists.txt new file mode 100644 index 000000000..2b8d9034f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/CMakeLists.txt @@ -0,0 +1,93 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +file(GLOB group_proto_src ${ASCEND_AUTOGEN_PATH}/group_proto/*.cc) + +add_library(cust_op_proto SHARED + $<$:${group_proto_src}> + ${ops_srcs} + ${ASCEND_AUTOGEN_PATH}/op_proto.cc +) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +file(GLOB GROUP_PROTO_HEADERS ${ASCEND_AUTOGEN_PATH}/group_proto/*.h) +if (GROUP_PROTO_HEADERS) + install(FILES ${GROUP_PROTO_HEADERS} + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +endif() +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp new file mode 100644 index 000000000..ddcf8818c --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp @@ -0,0 +1,61 @@ + +#include "demo_tiling.h" +#include "register/op_def_registry.h" + + +namespace optiling { +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + + DemoTilingData tiling; + uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); + + tiling.set_size(inputNum); + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class Demo : public OpDef { +public: + explicit Demo(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT16}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend910b"); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(Demo); +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h new file mode 100644 index 000000000..9dfd12fa5 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -0,0 +1,10 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(DemoTilingData) + TILING_DATA_FIELD_DEF(uint32_t, size); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) +} diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/CMakeLists.txt b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp new file mode 100644 index 000000000..7a94e4efa --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -0,0 +1,66 @@ +#include "kernel_operator.h" +constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue +class KernelDemo { +public: + __aicore__ inline KernelDemo() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, uint32_t totalLength) { + this->totalLength = totalLength; + xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); + yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); + pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); + + } + __aicore__ inline void Process() { + CopyIn(); + Compute(); + CopyOut(); + } + +private: + // TODO: user kernel private member + + __aicore__ inline void CopyIn(){ + AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); + AscendC::DataCopy(xLocal, xGm, this->totalLength); + inQueueX.EnQue(xLocal); + } + + __aicore__ inline void Compute(){ + AscendC::LocalTensor xLocal = inQueueX.DeQue(); + AscendC::LocalTensor yLocal = outQueueY.AllocTensor(); + float x = 0; + float y = 0; + half sum = 0; + for (size_t i = 0; i < this->totalLength; i++) + { + x= static_cast(xLocal.GetValue(i)); + y +=x; + sum =static_cast(y); + yLocal.SetValue(i, sum); + } + + + outQueueY.EnQue(yLocal); + inQueueX.FreeTensor(xLocal); + } + __aicore__ inline void CopyOut(){ + AscendC::LocalTensor yLocal = outQueueY.DeQue(); + AscendC::DataCopy(yGm, yLocal, this->totalLength); + outQueueY.FreeTensor(yLocal); + } +private: + AscendC::TPipe pipe; + AscendC::TQue inQueueX; + AscendC::TQue outQueueY; + AscendC::GlobalTensor xGm; + AscendC::GlobalTensor yGm; + uint32_t totalLength; +}; + +extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + KernelDemo op; + op.Init(x, y, tiling_data.size); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/install.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/install.sh new file mode 100755 index 000000000..b87a9790f --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + log "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + exit 1 + fi +fi +chmod u+w ${targetdir}/vendors + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}\nexport LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + log "[ERROR] echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ export LD_LIBRARY_PATH=${_ASCEND_CUSTOM_OPP_PATH}/op_api/lib/:\${LD_LIBRARY_PATH} ] to set the environment path" +fi + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/upgrade.sh b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/upgrade.sh new file mode 100755 index 000000000..65457d2dd --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/scripts/upgrade.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. + +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[ops_custom] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +if [ ! -x $targetdir ] || [ ! -w $targetdir ] || [ ! -r $targetdir ];then + log "[WARNING] The directory $targetdir does not have sufficient permissions. \ + Please check and modify the folder permissions (e.g., using chmod), \ + or use the --install-path option to specify an installation path and \ + change the environment variable ASCEND_CUSTOM_OPP_PATH to the specified path." +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + log "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +log "[INFO] upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +log "[INFO] upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/\<$vendor_name\>//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +echo "SUCCESS" +exit 0 diff --git a/operator_contrib/PrecisionTips/TypeConversion/README.md b/operator_contrib/PrecisionTips/TypeConversion/README.md new file mode 100644 index 000000000..2be829b95 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/README.md @@ -0,0 +1,173 @@ +## 概述 +该样例用于复现强制转换前后顺序导致的精度损失问题。 +## 目录结构介绍 +``` +├── TypeConversion //使用框架调用的方式调用Demo算子 +│ ├── AclNNInvocation // 通过aclnn调用的方式调用Demo算子 +│ ├── Demo // Demo精度异常算子工程 +│ ├── DemoFix // Demo精度正常算子工程 +│ └── Demo.json // Demo算子的原型定义json文件 +``` +## 算子工程介绍 +算子工程目录Demo包含算子实现的模板文件、编译脚本等，如下所示: +``` +├── Demo //Demo自定义算子工程 +│ ├── cmake +│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 +│ ├── op_host // host侧实现文件 +│ ├── op_kernel // kernel侧实现文件 +│ ├── scripts // 自定义算子工程打包相关脚本所在目录 +│ ├── build.sh // 编译入口脚本 +│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt +│ └── CMakePresets.json // 编译配置项 +``` +CANN软件包中提供了工程创建工具msopgen，Demo算子工程可通过Demo.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 +## 编译运行样例算子 +针对自定义算子工程，编译运行包含如下步骤： +- 编译自定义算子工程生成算子安装包； +- 安装自定义算子到算子库中； +- 调用执行自定义算子； + +详细操作如下所示。 +### 1. 获取源码包 +请参考[准备：获取样例代码](../README.md#codeready)完成源码包获取。 +### 2. 修改算子工程配置 + + + - 执行如下命令，切换到算子工程Demo目录。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/PrecisionTips/TypeConversion/Demo + ``` + + - 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + + - 执行如下命令，切换到算子工程DemoFix目录。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/PrecisionTips/TypeConversion/DemoFix + ``` + + - 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + + +### 3. 配置环境变量 + + 请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + - 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + - 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + - 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` +### 4. 调用执行算子工程 + +运行run_test.sh，即可复现错误，界面会输出 + ``` +*************** run Demo start**************** +生成测试数据 +INFO: generate input data success! +可执行不存在 +INFO: cmake success! +INFO: make success! +INFO: execute op! +INFO: acl executable run success! +first 10 mistakes data index: 000040, expected: 764.500000000, actual: 763.500000000, rdiff: 0.001308 +first 10 mistakes data index: 000041, expected: 779.000000000, actual: 778.000000000, rdiff: 0.001284 +first 10 mistakes data index: 000042, expected: 801.500000000, actual: 800.500000000, rdiff: 0.001247 +first 10 mistakes data index: 000043, expected: 804.500000000, actual: 803.500000000, rdiff: 0.001243 +first 10 mistakes data index: 000044, expected: 826.000000000, actual: 825.000000000, rdiff: 0.001210 +first 10 mistakes data index: 000045, expected: 848.000000000, actual: 847.000000000, rdiff: 0.001180 +first 10 mistakes data index: 000046, expected: 855.500000000, actual: 854.500000000, rdiff: 0.001169 +first 10 mistakes data index: 000047, expected: 860.500000000, actual: 859.500000000, rdiff: 0.001163 +first 10 mistakes data index: 000052, expected: 948.500000000, actual: 947.500000000, rdiff: 0.001055 +first 10 mistakes data index: 000053, expected: 952.500000000, actual: 951.500000000, rdiff: 0.001050 +first 10 mistakes data index: 000054, expected: 960.000000000, actual: 959.000000000, rdiff: 0.001041 +error ratio: 0.8633, tolrence: 0.0010 +[ERROR] result error +*************** run Demo end**************** +*************** run DemoFix start**************** +生成测试数据 +INFO: generate input data success! +可执行不存在 +INFO: cmake success! +INFO: make success! +INFO: execute op! +INFO: acl executable run success! +error ratio: 0.0000, tolrence: 0.0010 +test pass +*************** run DemoFix end**************** + +``` + +## 精度调优思路 +修改前代码: +``` + float x = 0; + float y = 0; + half sum = 0; + for (size_t i = 0; i < this->totalLength; i++) + { + x= static_cast(xLocal.GetValue(i)); + y= static_cast(sum); + x += y; + sum =static_cast(x); + yLocal.SetValue(i, sum); + } +``` +从half转float没有精度损失，从float转half有精度损失，所以sum在转换时损失了精度。而sum又作为下次运算的输入，所以精度损失会累积，导致精度下降。 + +修改后代码: +``` + float x = 0; + float y = 0; + half sum = 0; + for (size_t i = 0; i < this->totalLength; i++) + { + x= static_cast(xLocal.GetValue(i)); + y +=x; + sum =static_cast(y); + yLocal.SetValue(i, sum); + } +``` +修改后sum由y从float转half，精度损失，但下次参与运算的是float类型的y，所以精度损失不会累积。 \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/run_test.sh b/operator_contrib/PrecisionTips/TypeConversion/run_test.sh new file mode 100755 index 000000000..467a73309 --- /dev/null +++ b/operator_contrib/PrecisionTips/TypeConversion/run_test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +LOG_PATH=/var/log/npu/slog/debug +if [ -d "$LOG_PATH" ]; then + echo "清理日志，路径：$LOG_PATH" +else + LOG_PATH=~/ascend/log + echo "清理日志，路径：$LOG_PATH" +fi +rm -rf $LOG_PATH* + +current_path=$(pwd) +echo "*************** run Demo start****************" +cd Demo;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 +cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh +echo "*************** run Demo end****************" +cd $current_path +echo "*************** run DemoFix start****************" +cd DemoFix;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 +cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh +echo "*************** run DemoFix end****************" -- Gitee From 1957b3da2306ee46b75d0df60763886f2b616af5 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Tue, 21 Jan 2025 10:13:33 +0800 Subject: [PATCH 02/17] delete_block --- .../DifferentAlgorithm/Demo/op_host/demo.cpp | 11 ----------- .../DifferentAlgorithm/Demo/op_kernel/demo.cpp | 6 +----- .../DifferentAlgorithm/DemoFix/op_host/demo.cpp | 12 +----------- .../DifferentAlgorithm/DemoFix/op_kernel/demo.cpp | 6 +----- .../SequenceOperationSteps/Demo/op_host/demo.cpp | 10 ---------- .../SequenceOperationSteps/Demo/op_kernel/demo.cpp | 7 ++----- .../SequenceOperationSteps/DemoFix/op_host/demo.cpp | 12 +----------- .../DemoFix/op_kernel/demo.cpp | 3 --- .../TypeConversion/Demo/op_host/demo.cpp | 10 ---------- .../TypeConversion/Demo/op_kernel/demo.cpp | 7 +------ .../TypeConversion/DemoFix/op_host/demo.cpp | 12 +----------- 11 files changed, 8 insertions(+), 88 deletions(-) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp index ddcf8818c..d97b78904 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp @@ -1,12 +1,8 @@ - #include "demo_tiling.h" #include "register/op_def_registry.h" - - namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) { - DemoTilingData tiling; uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); @@ -18,8 +14,6 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) return ge::GRAPH_SUCCESS; } } - - namespace ge { static ge::graphStatus InferShape(gert::InferShapeContext* context) { @@ -29,8 +23,6 @@ static ge::graphStatus InferShape(gert::InferShapeContext* context) return GRAPH_SUCCESS; } } - - namespace ops { class Demo : public OpDef { public: @@ -48,14 +40,11 @@ public: .UnknownShapeFormat({ge::FORMAT_ND}); this->SetInferShape(ge::InferShape); - this->AICore() .SetTiling(optiling::TilingFunc); this->AICore().AddConfig("ascend910b"); this->AICore().AddConfig("ascend310b"); - } }; - OP_ADD(Demo); } diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp index 937e5211b..577daabfc 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -8,8 +8,7 @@ public: xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); - pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); - + pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); } __aicore__ inline void Process() { CopyIn(); @@ -19,19 +18,16 @@ public: private: // TODO: user kernel private member - __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::DataCopy(xLocal, xGm, this->totalLength); inQueueX.EnQue(xLocal); } - __aicore__ inline void Compute(){ AscendC::LocalTensor xLocal = inQueueX.DeQue(); AscendC::LocalTensor yLocal = outQueueY.AllocTensor(); // sqrt(2) ≈ 1.41 Muls(yLocal, xLocal, (half)1.41, this->totalLength); - outQueueY.EnQue(yLocal); inQueueX.FreeTensor(xLocal); } diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp index ddcf8818c..2e4e5d1e1 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp @@ -1,12 +1,8 @@ - #include "demo_tiling.h" #include "register/op_def_registry.h" - - namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) { - DemoTilingData tiling; uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); @@ -18,8 +14,6 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) return ge::GRAPH_SUCCESS; } } - - namespace ge { static ge::graphStatus InferShape(gert::InferShapeContext* context) { @@ -29,8 +23,6 @@ static ge::graphStatus InferShape(gert::InferShapeContext* context) return GRAPH_SUCCESS; } } - - namespace ops { class Demo : public OpDef { public: @@ -53,9 +45,7 @@ public: .SetTiling(optiling::TilingFunc); this->AICore().AddConfig("ascend910b"); this->AICore().AddConfig("ascend310b"); - } }; - OP_ADD(Demo); -} +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index b67d910b1..3f292250e 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -8,8 +8,7 @@ public: xGm.SetGlobalBuffer((__gm__ half *)x, this->totalLength); yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); - pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); - + pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); } __aicore__ inline void Process() { CopyIn(); @@ -19,13 +18,11 @@ public: private: // TODO: user kernel private member - __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::DataCopy(xLocal, xGm, this->totalLength); inQueueX.EnQue(xLocal); } - __aicore__ inline void Compute(){ AscendC::LocalTensor xLocal = inQueueX.DeQue(); AscendC::LocalTensor yLocal = outQueueY.AllocTensor(); @@ -49,7 +46,6 @@ private: AscendC::GlobalTensor yGm; uint32_t totalLength; }; - extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp index d8598f558..b48ed1b86 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp @@ -1,12 +1,8 @@ - #include "demo_tiling.h" #include "register/op_def_registry.h" - - namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) { - DemoTilingData tiling; uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); @@ -18,8 +14,6 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) return ge::GRAPH_SUCCESS; } } - - namespace ge { static ge::graphStatus InferShape(gert::InferShapeContext* context) { @@ -29,8 +23,6 @@ static ge::graphStatus InferShape(gert::InferShapeContext* context) return GRAPH_SUCCESS; } } - - namespace ops { class Demo : public OpDef { public: @@ -58,9 +50,7 @@ public: .SetTiling(optiling::TilingFunc); this->AICore().AddConfig("ascend910b"); this->AICore().AddConfig("ascend310b"); - } }; - OP_ADD(Demo); } diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index 7184389a5..67d9f9045 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -10,8 +10,7 @@ public: zGm.SetGlobalBuffer((__gm__ half *)z, this->totalLength); pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); pipe.InitBuffer(inQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); - pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->totalLength * sizeof(half)); - + pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->totalLength * sizeof(half)); } __aicore__ inline void Process() { CopyIn(); @@ -21,7 +20,6 @@ public: private: // TODO: user kernel private member - __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); @@ -37,8 +35,7 @@ private: AscendC::LocalTensor zLocal = outQueueZ.AllocTensor(); // 预期结果为 x *0.001+y*0.001 = (x + y) * 0.001 Add(zLocal, xLocal, yLocal, this->totalLength); - Muls(zLocal, zLocal, (half)0.001, this->totalLength); - + Muls(zLocal, zLocal, (half)0.001, this->totalLength); outQueueZ.EnQue(zLocal); inQueueX.FreeTensor(xLocal); inQueueY.FreeTensor(yLocal); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp index d8598f558..af35b24f8 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp @@ -1,12 +1,8 @@ - #include "demo_tiling.h" #include "register/op_def_registry.h" - - namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) { - DemoTilingData tiling; uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); @@ -18,8 +14,6 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) return ge::GRAPH_SUCCESS; } } - - namespace ge { static ge::graphStatus InferShape(gert::InferShapeContext* context) { @@ -29,8 +23,6 @@ static ge::graphStatus InferShape(gert::InferShapeContext* context) return GRAPH_SUCCESS; } } - - namespace ops { class Demo : public OpDef { public: @@ -58,9 +50,7 @@ public: .SetTiling(optiling::TilingFunc); this->AICore().AddConfig("ascend910b"); this->AICore().AddConfig("ascend310b"); - } }; - OP_ADD(Demo); -} +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index 221124607..bd2979ec6 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -18,10 +18,8 @@ public: Compute(); CopyOut(); } - private: // TODO: user kernel private member - __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); @@ -57,7 +55,6 @@ private: AscendC::GlobalTensor zGm; uint32_t totalLength; }; - extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp index ddcf8818c..9f6fdc2a4 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp @@ -1,12 +1,8 @@ - #include "demo_tiling.h" #include "register/op_def_registry.h" - - namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) { - DemoTilingData tiling; uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); @@ -18,8 +14,6 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) return ge::GRAPH_SUCCESS; } } - - namespace ge { static ge::graphStatus InferShape(gert::InferShapeContext* context) { @@ -29,8 +23,6 @@ static ge::graphStatus InferShape(gert::InferShapeContext* context) return GRAPH_SUCCESS; } } - - namespace ops { class Demo : public OpDef { public: @@ -53,9 +45,7 @@ public: .SetTiling(optiling::TilingFunc); this->AICore().AddConfig("ascend910b"); this->AICore().AddConfig("ascend310b"); - } }; - OP_ADD(Demo); } diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp index 2170ab50f..14458dc62 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -9,14 +9,12 @@ public: yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); - } __aicore__ inline void Process() { CopyIn(); Compute(); CopyOut(); } - private: // TODO: user kernel private member @@ -39,9 +37,7 @@ private: x += y; sum =static_cast(x); yLocal.SetValue(i, sum); - } - - + } outQueueY.EnQue(yLocal); inQueueX.FreeTensor(xLocal); } @@ -58,7 +54,6 @@ private: AscendC::GlobalTensor yGm; uint32_t totalLength; }; - extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp index ddcf8818c..2e4e5d1e1 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp @@ -1,12 +1,8 @@ - #include "demo_tiling.h" #include "register/op_def_registry.h" - - namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) { - DemoTilingData tiling; uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); @@ -18,8 +14,6 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) return ge::GRAPH_SUCCESS; } } - - namespace ge { static ge::graphStatus InferShape(gert::InferShapeContext* context) { @@ -29,8 +23,6 @@ static ge::graphStatus InferShape(gert::InferShapeContext* context) return GRAPH_SUCCESS; } } - - namespace ops { class Demo : public OpDef { public: @@ -53,9 +45,7 @@ public: .SetTiling(optiling::TilingFunc); this->AICore().AddConfig("ascend910b"); this->AICore().AddConfig("ascend310b"); - } }; - OP_ADD(Demo); -} +} \ No newline at end of file -- Gitee From 661efac6b4bc280c375710d9d9c69b0266da1db2 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Tue, 21 Jan 2025 10:36:46 +0800 Subject: [PATCH 03/17] update_no_understanding --- .../AddCustom/op_kernel/add_custom.cpp | 4 +-- .../AddKernelInvocation/add_custom.cpp | 3 ++- .../AddCustom/op_kernel/add_custom.cpp | 26 ++++++++++--------- .../AddKernelInvocation/main.cpp | 4 +-- .../AddKernelInvocation/main.cpp | 4 +-- .../AddCustom/op_kernel/add_custom.cpp | 8 +++--- .../AddCustom/op_host/add_custom.cpp | 7 ++--- .../AddKernelInvocation/main.cpp | 4 +-- .../DemoFix/op_kernel/demo.cpp | 4 +-- .../Demo/op_kernel/demo.cpp | 3 ++- .../DemoFix/op_kernel/demo.cpp | 5 ++-- 11 files changed, 39 insertions(+), 33 deletions(-) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp index 0d7ed7244..19c93c4e9 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp @@ -10,7 +10,6 @@ #include "kernel_operator.h" using namespace AscendC; constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue - class KernelAdd { public: __aicore__ inline KernelAdd() {} @@ -62,7 +61,7 @@ private: __aicore__ inline void CopyOut(int32_t progress) { LocalTensor zLocal = outQueueZ.DeQue(); - DataCopy(zGm[progress * this->tileLength + 128], zLocal, this->tileLength); + DataCopy(zGm[progress * this->tileLength + OFFSET], zLocal, this->tileLength); outQueueZ.FreeTensor(zLocal); } @@ -76,6 +75,7 @@ private: uint32_t blockLength; uint32_t tileNum; uint32_t tileLength; + constexpr int OFFSET = 128; }; extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp index daf334f89..0679b934f 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp @@ -62,7 +62,7 @@ private: __aicore__ inline void CopyOut(int32_t progress) { LocalTensor zLocal = outQueueZ.DeQue(); - DataCopy(zGm[progress * TILE_LENGTH+128], zLocal, TILE_LENGTH); + DataCopy(zGm[progress * TILE_LENGTH+OFFSET], zLocal, TILE_LENGTH); outQueueZ.FreeTensor(zLocal); } @@ -73,6 +73,7 @@ private: GlobalTensor xGm; GlobalTensor yGm; GlobalTensor zGm; + constexpr int OFFSET = 128; }; extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp index 62b57bd32..864eac266 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp @@ -7,7 +7,9 @@ #include "kernel_operator.h" using namespace AscendC; constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue - +constexpr int32_t BUFFER_SIZE = 4000; +constexpr int32_t DATA_SIZE = 2048; +constexpr int32_t OVER_SIZE = 4000*4000*8; class KernelAdd { public: __aicore__ inline KernelAdd() {} @@ -17,12 +19,12 @@ public: this->blockLength = totalLength / GetBlockNum(); this->tileNum = tileNum; ASSERT(tileNum != 0 && "tile num can not be zero!"); - xGm.SetGlobalBuffer((__gm__ DTYPE_X*)x , 2048); - yGm.SetGlobalBuffer((__gm__ DTYPE_Y*)y , 2048); - zGm.SetGlobalBuffer((__gm__ DTYPE_Z*)z ,2048); - pipe.InitBuffer(inQueueX, BUFFER_NUM, 4000 * sizeof(DTYPE_X)); - pipe.InitBuffer(inQueueY, BUFFER_NUM, 4000 * sizeof(DTYPE_Y)); - pipe.InitBuffer(outQueueZ, BUFFER_NUM, 4000 * sizeof(DTYPE_Z)); + xGm.SetGlobalBuffer((__gm__ DTYPE_X*)x , DATA_SIZE); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y*)y , DATA_SIZE); + zGm.SetGlobalBuffer((__gm__ DTYPE_Z*)z ,DATA_SIZE); + pipe.InitBuffer(inQueueX, BUFFER_NUM, BUFFER_SIZE * sizeof(DTYPE_X)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, BUFFER_SIZE * sizeof(DTYPE_Y)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, BUFFER_SIZE * sizeof(DTYPE_Z)); } __aicore__ inline void Process() { @@ -39,8 +41,8 @@ private: { LocalTensor xLocal = inQueueX.AllocTensor(); LocalTensor yLocal = inQueueY.AllocTensor(); - DataCopy(xLocal, xGm[progress * 2048], 2048); - DataCopy(yLocal, yGm[progress * 2048], 2048); + DataCopy(xLocal, xGm[progress * DATA_SIZE], DATA_SIZE); + DataCopy(yLocal, yGm[progress * DATA_SIZE], DATA_SIZE); inQueueX.EnQue(xLocal); inQueueY.EnQue(yLocal); } @@ -49,7 +51,7 @@ private: LocalTensor xLocal = inQueueX.DeQue(); LocalTensor yLocal = inQueueY.DeQue(); LocalTensor zLocal = outQueueZ.AllocTensor(); - Add(zLocal, xLocal, yLocal, 2048); + Add(zLocal, xLocal, yLocal, DATA_SIZE); outQueueZ.EnQue(zLocal); inQueueX.FreeTensor(xLocal); inQueueY.FreeTensor(yLocal); @@ -57,8 +59,8 @@ private: __aicore__ inline void CopyOut(int32_t progress) { LocalTensor zLocal = outQueueZ.DeQue(); - DataCopy(zGm, zLocal, 2048); - zGm.SetValue(4000*4000*8,zLocal.GetValue(0)); + DataCopy(zGm, zLocal, DATA_SIZE); + zGm.SetValue(OVER_SIZE,zLocal.GetValue(0)); outQueueZ.FreeTensor(zLocal); } private: diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp index bd760f1c6..75f01c7db 100644 --- a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp @@ -19,8 +19,8 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z int32_t main(int32_t argc, char *argv[]) { uint32_t blockDim = 1; - size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); - size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t inputByteSize = static_cast(8)* static_cast(2048) * sizeof(uint16_t); + size_t outputByteSize = static_cast(8)* static_cast(2048) * sizeof(uint16_t); #ifdef ASCENDC_CPU_DEBUG uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp index bd760f1c6..75f01c7db 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp @@ -19,8 +19,8 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z int32_t main(int32_t argc, char *argv[]) { uint32_t blockDim = 1; - size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); - size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t inputByteSize = static_cast(8)* static_cast(2048) * sizeof(uint16_t); + size_t outputByteSize = static_cast(8)* static_cast(2048) * sizeof(uint16_t); #ifdef ASCENDC_CPU_DEBUG uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp index 03c0c5a00..e57ec4ff7 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp @@ -7,7 +7,8 @@ #include "kernel_operator.h" using namespace AscendC; constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue - +constexpr int32_t CIRCLE_NUM = 202; // circle num +constexpr int32_t OVER_SIZE = 4000*4000*8; // circle num class KernelAdd { public: @@ -56,7 +57,7 @@ public: Add(zLocal, xLocal, yLocal, this->tileLength); outQueueZ.EnQue(zLocal); - for (int32_t k = 0; k < 202; k++) + for (int32_t k = 0; k < CIRCLE_NUM; k++) { zLocal = outQueueZ.DeQue(); outQueueZ.EnQue(zLocal); @@ -65,7 +66,6 @@ public: inQueueX.FreeTensor(xLocal); inQueueY.FreeTensor(yLocal); zLocal = outQueueZ.DeQue(); - // DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); DataCopy(zGm[i * this->tileLength], zLocal, this->tileLength); outQueueZ.FreeTensor(zLocal); @@ -100,7 +100,7 @@ private: // DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); - zGm.SetValue(4000*4000*8,zLocal.GetValue(0)); + zGm.SetValue(OVER_SIZE,zLocal.GetValue(0)); outQueueZ.FreeTensor(zLocal); } diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp index 870b01d62..0c01e9c64 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp @@ -20,13 +20,14 @@ static ge::graphStatus TilingFunc(gert::TilingContext *context) context->SetBlockDim(BLOCK_DIM); tiling.set_totalLength(totalLength); tiling.set_tileNum(TILE_NUM); - + constexpr int INPUT_SHAPE_COUNT = 2; int64_t shape[6]; - for (uint32_t i = 0; i < 2; i++) { + for (uint32_t i = 0; i < INPUT_SHAPE_COUNT; i++) { int64_t *ss = &shape[i * 3]; const gert::StorageShape* storageShape = context -> GetInputShape(i); for (int j = 0; j < storageShape->GetStorageShape().GetDimNum(); j++) { - ss[2 * i + j] = storageShape->GetStorageShape().GetDim(j); + constexpr int DIM_OFFSET = 2; + ss[DIM_OFFSET * i + j] = storageShape->GetStorageShape().GetDim(j); } } tiling.set_shape(shape); diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp index bd760f1c6..75f01c7db 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp @@ -19,8 +19,8 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z int32_t main(int32_t argc, char *argv[]) { uint32_t blockDim = 1; - size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); - size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t inputByteSize = static_cast(8)* static_cast(2048) * sizeof(uint16_t); + size_t outputByteSize = static_cast(8)* static_cast(2048) * sizeof(uint16_t); #ifdef ASCENDC_CPU_DEBUG uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index 3f292250e..c2651354b 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -26,8 +26,8 @@ private: __aicore__ inline void Compute(){ AscendC::LocalTensor xLocal = inQueueX.DeQue(); AscendC::LocalTensor yLocal = outQueueY.AllocTensor(); - half inputVal(2.0); - AscendC::Duplicate(yLocal, inputVal, this->totalLength); + constexpr half INPUT_VAL = 2.0; + AscendC::Duplicate(yLocal, INPUT_VAL, this->totalLength); AscendC::Sqrt(yLocal,yLocal,this->totalLength); Mul(yLocal, xLocal, yLocal, this->totalLength); outQueueY.EnQue(yLocal); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index 67d9f9045..48fc304c2 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -34,8 +34,9 @@ private: AscendC::LocalTensor yLocal = inQueueY.DeQue(); AscendC::LocalTensor zLocal = outQueueZ.AllocTensor(); // 预期结果为 x *0.001+y*0.001 = (x + y) * 0.001 + constexpr half INPUT_VAL = 0.001; Add(zLocal, xLocal, yLocal, this->totalLength); - Muls(zLocal, zLocal, (half)0.001, this->totalLength); + Muls(zLocal, zLocal, INPUT_VAL, this->totalLength); outQueueZ.EnQue(zLocal); inQueueX.FreeTensor(xLocal); inQueueY.FreeTensor(yLocal); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index bd2979ec6..cab4867e9 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -33,8 +33,9 @@ private: AscendC::LocalTensor xLocal = inQueueX.DeQue(); AscendC::LocalTensor yLocal = inQueueY.DeQue(); AscendC::LocalTensor zLocal = outQueueZ.AllocTensor(); - Muls(xLocal, xLocal, (half)0.001, this->totalLength); - Muls(yLocal, yLocal, (half)0.001, this->totalLength); + constexpr half INPUT_VAL = 0.001; + Muls(xLocal, xLocal, INPUT_VAL, this->totalLength); + Muls(yLocal, yLocal, INPUT_VAL, this->totalLength); Add(zLocal, xLocal, yLocal, this->totalLength); outQueueZ.EnQue(zLocal); -- Gitee From a0b348b869cc3a003770035d389c75d9615c957c Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Tue, 21 Jan 2025 11:05:50 +0800 Subject: [PATCH 04/17] update_Licensed --- .../cceprint/add_custom_0_0_mix.cce | 117 ------------------ .../cceprint/add_custom_0_0_mix.cce | 15 --- .../AddCustom/op_host/add_custom_cc.cpp | 16 +-- .../AddCustom/op_host/add_custom_cc_tiling.h | 14 ++- .../AddCustom/op_kernel/add_custom1.cpp | 11 +- .../SegmentationFault/tiling_debug.py | 9 ++ .../AclNNInvocation/scripts/get_time.py | 15 ++- .../DifferentAlgorithm/Demo/op_host/demo.cpp | 11 +- .../Demo/op_host/demo_tiling.h | 13 +- .../Demo/op_kernel/demo.cpp | 11 +- .../DemoFix/op_host/demo.cpp | 11 +- .../DemoFix/op_host/demo_tiling.h | 13 +- .../DemoFix/op_kernel/demo.cpp | 9 ++ .../Demo/op_host/demo.cpp | 9 ++ .../Demo/op_host/demo_tiling.h | 13 +- .../Demo/op_kernel/demo.cpp | 9 ++ .../DemoFix/op_host/demo.cpp | 9 ++ .../DemoFix/op_host/demo_tiling.h | 10 +- .../DemoFix/op_kernel/demo.cpp | 9 ++ .../TypeConversion/Demo/op_host/demo.cpp | 9 ++ .../TypeConversion/Demo/op_host/demo_tiling.h | 10 +- .../TypeConversion/Demo/op_kernel/demo.cpp | 9 ++ .../TypeConversion/DemoFix/op_host/demo.cpp | 9 ++ .../DemoFix/op_host/demo_tiling.h | 10 +- .../TypeConversion/DemoFix/op_kernel/demo.cpp | 9 ++ 25 files changed, 214 insertions(+), 166 deletions(-) delete mode 100644 operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce delete mode 100644 operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce deleted file mode 100644 index 6bdd65bd9..000000000 --- a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce +++ /dev/null @@ -1,117 +0,0 @@ -extern "C" __global__ [aicore] void add_custom(__gm__ uint8_t* __restrict__ arg0, __gm__ uint8_t* __restrict__ arg1, __gm__ uint8_t* __restrict__ arg2) { -set_ffts_base_addr((uint64_t)0xffffa8a896a0); -set_atomic_none(); -set_padding((uint64_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)1); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)2); -copy_gm_to_ubuf((__ubuf__ void*)0xffffa0844010, (__gm__ void*)arg0, (uint8_t)0, (uint16_t)1, (uint16_t)128, (uint16_t)0, (uint16_t)0); -copy_gm_to_ubuf((__ubuf__ void*)0xffffa0847e90, (__gm__ void*)arg1, (uint8_t)0, (uint16_t)1, (uint16_t)128, (uint16_t)0, (uint16_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)1); -wait_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)0); -#error "sbitset0 NO PROCESS RETURN VALUE" -wait_flag((pipe_t)PIPE_MTE2, (pipe_t)PIPE_V, (event_t)1); -#error "sbitset0 NO PROCESS RETURN VALUE" -plt_b16((uint32_t &)2048, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)0, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)0, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)1, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)0, (Literal)4, (vector_bool)1); -plt_b16((uint32_t &)1920, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)128, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)128, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)2, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)128, (Literal)4, (vector_bool)2); -plt_b16((uint32_t &)1792, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)256, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)256, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)3, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)256, (Literal)4, (vector_bool)3); -plt_b16((uint32_t &)1664, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)384, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)384, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)4, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)384, (Literal)4, (vector_bool)4); -plt_b16((uint32_t &)1536, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)512, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)512, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)5, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)512, (Literal)4, (vector_bool)5); -plt_b16((uint32_t &)1408, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)640, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)640, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)6, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)640, (Literal)4, (vector_bool)6); -plt_b16((uint32_t &)1280, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)768, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)768, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)7, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)768, (Literal)4, (vector_bool)7); -plt_b16((uint32_t &)1152, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)896, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)896, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)8, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)896, (Literal)4, (vector_bool)8); -plt_b16((uint32_t &)1024, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1024, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1024, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)9, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1024, (Literal)4, (vector_bool)9); -plt_b16((uint32_t &)896, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1152, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1152, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)10, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1152, (Literal)4, (vector_bool)10); -plt_b16((uint32_t &)768, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1280, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1280, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)11, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1280, (Literal)4, (vector_bool)11); -plt_b16((uint32_t &)640, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1408, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1408, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)12, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1408, (Literal)4, (vector_bool)12); -plt_b16((uint32_t &)512, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1536, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1536, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)13, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1536, (Literal)4, (vector_bool)13); -plt_b16((uint32_t &)384, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1664, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1664, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)14, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1664, (Literal)4, (vector_bool)14); -plt_b16((uint32_t &)256, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1792, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1792, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)15, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1792, (Literal)4, (vector_bool)15); -plt_b16((uint32_t &)128, (Literal)1); -vlds((vector_f16 &)0xffffc15a86a0, (__ubuf__ half *)0xffffa0844010, (int32_t)1920, (Literal)2); -vlds((vector_f16 &)0xffffc15a87a0, (__ubuf__ half *)0xffffa0847e90, (int32_t)1920, (Literal)2); -vadd((vector_f16 &)0xffffc15a88a0, (vector_f16)0xffffc15a86a0, (vector_f16)0xffffc15a87a0, (vector_bool)16, (Literal)0); -vsts((vector_f16)0xffffc15a88a0, (__ubuf__ half *)0xffffa084bd10, (int32_t)1920, (Literal)4, (vector_bool)16); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE3, (event_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE2, (event_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE2, (event_t)1); -wait_flag((pipe_t)PIPE_V, (pipe_t)PIPE_MTE3, (event_t)0); -#error "sbitset0 NO PROCESS RETURN VALUE" -copy_ubuf_to_gm((__gm__ void*)arg2, (__ubuf__ void*)0xffffa084bd10, (uint8_t)0, (uint16_t)1, (uint16_t)128, (uint16_t)0, (uint16_t)0); diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce deleted file mode 100644 index 2c5d04cb7..000000000 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/cceprint/add_custom_0_0_mix.cce +++ /dev/null @@ -1,15 +0,0 @@ -extern "C" __global__ [aicore] void add_custom(__gm__ uint8_t* __restrict__ arg0, __gm__ uint8_t* __restrict__ arg1, __gm__ uint8_t* __restrict__ arg2) { -set_ffts_base_addr((uint64_t)0xffff931e96a0); -set_atomic_none(); -set_padding((uint64_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)0); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)1); -#error "sff0 NO PROCESS RETURN VALUE" -#error "sbitset1 NO PROCESS RETURN VALUE" -set_flag((pipe_t)PIPE_M, (pipe_t)PIPE_MTE1, (event_t)2); -copy_gm_to_ubuf((__ubuf__ void*)0xffff8afa4010, (__gm__ void*)arg0, (uint8_t)0, (uint16_t)1, (uint16_t)64, (uint16_t)0, (uint16_t)0); -copy_gm_to_ubuf((__ubuf__ void*)0xffff8afa5010, (__gm__ void*)arg1, (uint8_t)0, (uint16_t)1, (uint16_t)64, (uint16_t)0, (uint16_t)0); diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp index 4dc5bcbde..e46d31c79 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp @@ -1,8 +1,14 @@ - +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "add_custom_cc_tiling.h" #include "register/op_def_registry.h" - - namespace optiling { const uint32_t BLOCK_DIM = 8; const uint32_t TILE_NUM = 8; @@ -20,8 +26,6 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) return ge::GRAPH_SUCCESS; } } - - namespace ge { static ge::graphStatus InferShape(gert::InferShapeContext* context) { @@ -31,8 +35,6 @@ static ge::graphStatus InferShape(gert::InferShapeContext* context) return GRAPH_SUCCESS; } } - - namespace ops { class AddCustom1 : public OpDef { public: diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index 8b9058ce5..70c169bc2 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -1,11 +1,17 @@ - +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" - namespace optiling { BEGIN_TILING_DATA_DEF(AddCustomCCTilingData) TILING_DATA_FIELD_DEF(uint32_t, totalLength); TILING_DATA_FIELD_DEF(uint32_t, tileNum); END_TILING_DATA_DEF; - REGISTER_TILING_DATA_CLASS(AddCustom1, AddCustomCCTilingData) -} +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index 4b4ca7611..133b4ac09 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -1,8 +1,15 @@ - +/** + * @file add_custom_tiling.h + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "kernel_operator.h" using namespace AscendC; constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue - class KernelAdd { public: __aicore__ inline KernelAdd() {} diff --git a/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py b/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py index ec9ee641f..86d6f3c61 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py +++ b/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py @@ -1,3 +1,12 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== import ascendebug import numpy as np input_x = np.random.uniform(1, 100, [8, 2048]).astype(np.float16) diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py index fbeac398d..b6214ef2d 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py @@ -1,7 +1,14 @@ +#!/usr/bin/python3 +# coding=utf-8 +# +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# =============================================================================== import csv from pathlib import Path - - def get_time(file_path, time_use_list): with open(file_path, 'r', encoding='utf-8') as file: reader = csv.DictReader(file) @@ -9,7 +16,6 @@ def get_time(file_path, time_use_list): time_use = row['Task Duration(us)'] time_use_list.append(int(float(time_use)* 1000000)) - def find_min_time(): min_time = 0 time_use_list = [] @@ -26,6 +32,3 @@ def find_min_time(): if __name__ == '__main__': find_min_time() - - - diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp index d97b78904..4a69d0236 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "demo_tiling.h" #include "register/op_def_registry.h" namespace optiling { @@ -47,4 +56,4 @@ public: } }; OP_ADD(Demo); -} +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h index 9dfd12fa5..c661fdbbd 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -1,10 +1,17 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" - namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) -} +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp index 577daabfc..fe4be7923 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -1,4 +1,13 @@ -#include "kernel_operator.h" +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ + #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue class KernelDemo { public: diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp index 2e4e5d1e1..83c058814 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp @@ -1,4 +1,13 @@ -#include "demo_tiling.h" +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * + /#include "demo_tiling.h" #include "register/op_def_registry.h" namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h index 9dfd12fa5..c661fdbbd 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -1,10 +1,17 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" - namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) -} +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index c2651354b..a1d88fd64 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue class KernelDemo { diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp index b48ed1b86..5c51b8e30 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "demo_tiling.h" #include "register/op_def_registry.h" namespace optiling { diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h index 9dfd12fa5..c661fdbbd 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -1,10 +1,17 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" - namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) -} +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index 48fc304c2..45d16d67f 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue class KernelDemo { diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp index af35b24f8..3df8fa918 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "demo_tiling.h" #include "register/op_def_registry.h" namespace optiling { diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h index 9dfd12fa5..a1953af99 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -1,4 +1,12 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" namespace optiling { diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index cab4867e9..df1f36a3f 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue class KernelDemo { diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp index 9f6fdc2a4..7c1a74d72 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "demo_tiling.h" #include "register/op_def_registry.h" namespace optiling { diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h index 9dfd12fa5..a1953af99 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -1,4 +1,12 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" namespace optiling { diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp index 14458dc62..a2ab15a30 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue class KernelDemo { diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp index 2e4e5d1e1..dce317aef 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "demo_tiling.h" #include "register/op_def_registry.h" namespace optiling { diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h index 9dfd12fa5..a1953af99 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -1,4 +1,12 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" namespace optiling { diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp index 7a94e4efa..16216d759 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -1,3 +1,12 @@ +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue class KernelDemo { -- Gitee From 9415eea35ac29e18636ae5347520e16c4a158c39 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Tue, 21 Jan 2025 11:50:10 +0800 Subject: [PATCH 05/17] update_declaration --- .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../AddKernelInvocation/add_custom.cpp | 3 +- .../AddKernelInvocation/main.cpp | 9 ++++- .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../AddKernelInvocation/add_custom.cpp | 3 +- .../AddKernelInvocation/main.cpp | 9 ++++- .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../AddKernelInvocation/add_custom.cpp | 37 ++++++++++++------- .../AddKernelInvocation/main.cpp | 9 ++++- .../AddCustom/op_kernel/add_custom.cpp | 4 +- .../AddKernelInvocation/add_custom.cpp | 4 +- .../AddKernelInvocation/main.cpp | 9 ++++- .../AddCustom/op_kernel/add_custom.cpp | 6 +-- .../AddCustom/op_kernel/add_custom1.cpp | 5 +-- .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../AddCustom/op_kernel/add_custom.cpp | 4 +- .../AddKernelInvocation/add_custom.cpp | 1 - .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../AddCustom/op_kernel/add_custom.cpp | 3 +- .../Demo/op_kernel/demo.cpp | 3 +- .../DemoFix/op_kernel/demo.cpp | 3 +- 23 files changed, 85 insertions(+), 48 deletions(-) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp index 19c93c4e9..eab8ce944 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp @@ -67,7 +67,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp index 0679b934f..b8e5524ce 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp @@ -68,7 +68,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp index bd760f1c6..3bfdff916 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp @@ -45,8 +45,13 @@ int32_t main(int32_t argc, char *argv[]) aclrtStream stream = nullptr; CHECK_ACL(aclrtCreateStream(&stream)); - uint8_t *xHost, *yHost, *zHost; - uint8_t *xDevice, *yDevice, *zDevice; + uint8_t *xHost; + uint8_t *yHost; + uint8_t *zHost; + + uint8_t *xDevice; + uint8_t *yDevice; + uint8_t *zDevice; CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp index 6d2c1346c..4c21ef57c 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp @@ -68,7 +68,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp index d77c14b83..815ae452b 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp @@ -68,7 +68,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp index bd760f1c6..3bfdff916 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp @@ -45,8 +45,13 @@ int32_t main(int32_t argc, char *argv[]) aclrtStream stream = nullptr; CHECK_ACL(aclrtCreateStream(&stream)); - uint8_t *xHost, *yHost, *zHost; - uint8_t *xDevice, *yDevice, *zDevice; + uint8_t *xHost; + uint8_t *yHost; + uint8_t *zHost; + + uint8_t *xDevice; + uint8_t *yDevice; + uint8_t *zDevice; CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp index 864eac266..4f377c719 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp @@ -65,7 +65,8 @@ private: } private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp index d2f17a227..315d74dda 100644 --- a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp @@ -7,7 +7,9 @@ #include "kernel_operator.h" using namespace AscendC; constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue - +constexpr int32_t BUFFER_SIZE = 4000; +constexpr int32_t DATA_SIZE = 2048; +constexpr int32_t OVER_SIZE = BUFFER_SIZE*BUFFER_SIZE*8; class KernelAdd { public: __aicore__ inline KernelAdd() {} @@ -17,12 +19,12 @@ public: this->blockLength = totalLength / GetBlockNum(); this->tileNum = tileNum; ASSERT(tileNum != 0 && "tile num can not be zero!"); - xGm.SetGlobalBuffer((__gm__ half*)x , 2048); - yGm.SetGlobalBuffer((__gm__ half*)y , 2048); - zGm.SetGlobalBuffer((__gm__ half*)z ,2048); - pipe.InitBuffer(inQueueX, BUFFER_NUM, 4000 * sizeof(half)); - pipe.InitBuffer(inQueueY, BUFFER_NUM, 4000 * sizeof(half)); - pipe.InitBuffer(outQueueZ, BUFFER_NUM, 4000 * sizeof(half)); + xGm.SetGlobalBuffer((__gm__ half*)x , DATA_SIZE); + yGm.SetGlobalBuffer((__gm__ half*)y , DATA_SIZE); + zGm.SetGlobalBuffer((__gm__ half*)z ,DATA_SIZE); + pipe.InitBuffer(inQueueX, BUFFER_NUM, BUFFER_SIZE * sizeof(half)); + pipe.InitBuffer(inQueueY, BUFFER_NUM, BUFFER_SIZE * sizeof(half)); + pipe.InitBuffer(outQueueZ, BUFFER_NUM, BUFFER_SIZE * sizeof(half)); } __aicore__ inline void Process() { @@ -38,8 +40,8 @@ private: { LocalTensor xLocal = inQueueX.AllocTensor(); LocalTensor yLocal = inQueueY.AllocTensor(); - DataCopy(xLocal, xGm[progress * 2048], 2048); - DataCopy(yLocal, yGm[progress * 2048], 2048); + DataCopy(xLocal, xGm[progress * DATA_SIZE], DATA_SIZE); + DataCopy(yLocal, yGm[progress * DATA_SIZE], DATA_SIZE); inQueueX.EnQue(xLocal); inQueueY.EnQue(yLocal); } @@ -48,7 +50,7 @@ private: LocalTensor xLocal = inQueueX.DeQue(); LocalTensor yLocal = inQueueY.DeQue(); LocalTensor zLocal = outQueueZ.AllocTensor(); - Add(zLocal, xLocal, yLocal, 2048); + Add(zLocal, xLocal, yLocal, DATA_SIZE); outQueueZ.EnQue(zLocal); inQueueX.FreeTensor(xLocal); @@ -58,14 +60,15 @@ private: { LocalTensor zLocal = outQueueZ.DeQue(); - DataCopy(zGm, zLocal, 2048); - zGm.SetValue(4000*4000*8,zLocal.GetValue(0)); + DataCopy(zGm, zLocal, DATA_SIZE); + zGm.SetValue(OVER_SIZE,zLocal.GetValue(0)); outQueueZ.FreeTensor(zLocal); } private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; @@ -80,4 +83,10 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z KernelAdd op; op.Init(x, y, z, 16384, 8); op.Process(); -} \ No newline at end of file +} +#ifndef ASCENDC_CPU_DEBUG +void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +{ + add_custom<<>>(x, y, z); +} +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp index 75f01c7db..036cd6505 100644 --- a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/main.cpp @@ -45,8 +45,13 @@ int32_t main(int32_t argc, char *argv[]) aclrtStream stream = nullptr; CHECK_ACL(aclrtCreateStream(&stream)); - uint8_t *xHost, *yHost, *zHost; - uint8_t *xDevice, *yDevice, *zDevice; + uint8_t *xHost; + uint8_t *yHost; + uint8_t *zHost; + + uint8_t *xDevice; + uint8_t *yDevice; + uint8_t *zDevice; CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp index 1e6a83fba..501ee0860 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp @@ -25,7 +25,6 @@ public: xGm.SetGlobalBuffer((__gm__ DTYPE_X *)x + this->blockLength * GetBlockIdx(), this->blockLength); yGm.SetGlobalBuffer((__gm__ DTYPE_Y *)y + this->blockLength * GetBlockIdx(), this->blockLength); zGm.SetGlobalBuffer((__gm__ DTYPE_Z *)z + this->blockLength * GetBlockIdx(), this->blockLength); - // pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileLength * sizeof(DTYPE_X)); pipe.InitBuffer(inQueueY, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Y)); pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->tileLength * sizeof(DTYPE_Z)); } @@ -68,7 +67,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp index 4efceda1a..b9906a3d8 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp @@ -25,7 +25,6 @@ public: xGm.SetGlobalBuffer((__gm__ half *)x + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); yGm.SetGlobalBuffer((__gm__ half *)y + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); zGm.SetGlobalBuffer((__gm__ half *)z + BLOCK_LENGTH * GetBlockIdx(), BLOCK_LENGTH); - // pipe.InitBuffer(inQueueX, BUFFER_NUM, TILE_LENGTH * sizeof(half)); pipe.InitBuffer(inQueueY, BUFFER_NUM, TILE_LENGTH * sizeof(half)); pipe.InitBuffer(outQueueZ, BUFFER_NUM, TILE_LENGTH * sizeof(half)); } @@ -68,7 +67,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp index 75f01c7db..036cd6505 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/main.cpp @@ -45,8 +45,13 @@ int32_t main(int32_t argc, char *argv[]) aclrtStream stream = nullptr; CHECK_ACL(aclrtCreateStream(&stream)); - uint8_t *xHost, *yHost, *zHost; - uint8_t *xDevice, *yDevice, *zDevice; + uint8_t *xHost; + uint8_t *yHost; + uint8_t *zHost; + + uint8_t *xDevice; + uint8_t *yDevice; + uint8_t *zDevice; CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp index e57ec4ff7..ebd7a54bb 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp @@ -66,7 +66,6 @@ public: inQueueX.FreeTensor(xLocal); inQueueY.FreeTensor(yLocal); zLocal = outQueueZ.DeQue(); - // DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); DataCopy(zGm[i * this->tileLength], zLocal, this->tileLength); outQueueZ.FreeTensor(zLocal); } @@ -97,8 +96,6 @@ private: __aicore__ inline void CopyOut(int32_t progress) { LocalTensor zLocal = outQueueZ.DeQue(); - - // DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); zGm.SetValue(OVER_SIZE,zLocal.GetValue(0)); outQueueZ.FreeTensor(zLocal); @@ -106,7 +103,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index 133b4ac09..577408694 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -54,7 +54,6 @@ private: LocalTensor yLocal = inQueueY.DeQue(); LocalTensor zLocal = outQueueZ.AllocTensor(); Add(zLocal, xLocal, yLocal, this->tileLength); - //DataCopy(zLocal,xLocal,this->tileLength); outQueueZ.EnQue(zLocal); inQueueX.FreeTensor(xLocal); inQueueY.FreeTensor(yLocal); @@ -62,14 +61,14 @@ private: __aicore__ inline void CopyOut(int32_t progress) { LocalTensor zLocal = outQueueZ.DeQue(); - //DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); outQueueZ.FreeTensor(zLocal); } private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp index 2685319f6..f432a3616 100644 --- a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp @@ -65,7 +65,8 @@ private: private: AscendC::TPipe pipe; - AscendC::TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; AscendC::TQue outQueueZ; AscendC::GlobalTensor xGm; AscendC::GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp index 584092996..a1efbf848 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp @@ -68,7 +68,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp index e3cd2837d..f5760c5cd 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp @@ -64,12 +64,12 @@ private: { LocalTensor zLocal = outQueueZ.DeQue(); DataCopy(zGm[progress * this->tileLength], zLocal, this->tileLength); - // outQueueZ.FreeTensor(zLocal); } private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp index 9ee4fb6df..bdf41a229 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp @@ -63,7 +63,6 @@ private: { LocalTensor zLocal = outQueueZ.DeQue(); DataCopy(zGm[progress * TILE_LENGTH], zLocal, TILE_LENGTH); - // outQueueZ.FreeTensor(zLocal); } private: diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp index 405cc38dd..629577ca2 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp @@ -71,7 +71,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp index 07a286690..384b915b7 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp @@ -65,7 +65,8 @@ private: private: AscendC::TPipe pipe; - AscendC::TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; AscendC::TQue outQueueZ; AscendC::GlobalTensor xGm; AscendC::GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp index 584092996..a1efbf848 100644 --- a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp @@ -68,7 +68,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index 45d16d67f..16d1b421e 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -57,7 +57,8 @@ private: } private: AscendC::TPipe pipe; - AscendC::TQue inQueueX,inQueueY; + TQue inQueueX; + TQue inQueueY; AscendC::TQue outQueueZ; AscendC::GlobalTensor xGm; AscendC::GlobalTensor yGm; diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index df1f36a3f..b0fe3954e 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -58,7 +58,8 @@ private: } private: AscendC::TPipe pipe; - AscendC::TQue inQueueX,inQueueY; + TQue inQueueX; + TQue inQueueY; AscendC::TQue outQueueZ; AscendC::GlobalTensor xGm; AscendC::GlobalTensor yGm; -- Gitee From cea0dd5773d6aaf9bf048ffb707e097c353da92e Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Tue, 21 Jan 2025 14:52:36 +0800 Subject: [PATCH 06/17] update_correct --- .../AclTensorFreeFail/AclNNInvocation/src/common.cpp | 2 +- .../AddCustom/op_kernel/add_custom.cpp | 11 +++++++++-- .../AddKernelInvocation/add_custom.cpp | 9 +++++++-- .../AddCustom/op_kernel/add_custom.cpp | 9 ++++++++- .../InvalidMemory/AclNNInvocation/src/common.cpp | 2 +- .../InvalidMemory/AddCustom/op_kernel/add_custom.cpp | 9 ++++++++- .../AddCustom/op_kernel/add_custom.cpp | 9 ++++++++- .../NoTilingKey/AclNNInvocation/src/common.cpp | 2 +- .../NoTilingKey/AddCustom/op_kernel/add_custom.cpp | 10 +++++++++- .../OpNameError/AclNNInvocation/src/common.cpp | 2 +- .../AddCustom/op_host/add_custom_cc_tiling.h | 5 ++++- .../OpNameError/AddCustom/op_kernel/add_custom1.cpp | 11 ++++++++++- .../ResultError/AddCustom/op_kernel/add_custom.cpp | 10 +++++++++- .../AddCustom/op_host/add_custom.cpp | 2 +- .../AddCustom/op_kernel/add_custom.cpp | 10 +++++++++- .../AddCustom/op_kernel/add_custom.cpp | 10 +++++++++- .../TimeOut/AclNNInvocation/src/common.cpp | 2 +- .../TimeOut/AddCustom/op_kernel/add_custom.cpp | 10 +++++++++- .../AddCustom/op_kernel/add_custom.cpp | 10 +++++++++- .../TypeDeletion/AddCustom/op_kernel/add_custom.cpp | 10 +++++++++- .../DifferentAlgorithm/AclNNInvocation/src/common.cpp | 2 +- .../DifferentAlgorithm/Demo/op_host/demo_tiling.h | 5 ++++- .../DifferentAlgorithm/Demo/op_kernel/demo.cpp | 2 +- .../DifferentAlgorithm/DemoFix/op_host/demo_tiling.h | 5 ++++- .../DifferentAlgorithm/DemoFix/op_kernel/demo.cpp | 1 - .../AclNNInvocation/src/common.cpp | 2 +- .../SequenceOperationSteps/Demo/op_host/demo_tiling.h | 5 ++++- .../SequenceOperationSteps/Demo/op_kernel/demo.cpp | 1 - .../DemoFix/op_host/demo_tiling.h | 4 +++- .../SequenceOperationSteps/DemoFix/op_kernel/demo.cpp | 1 - .../TypeConversion/AclNNInvocation/src/common.cpp | 2 +- .../TypeConversion/Demo/op_host/demo_tiling.h | 4 +++- .../TypeConversion/Demo/op_kernel/demo.cpp | 2 -- .../TypeConversion/DemoFix/op_host/demo_tiling.h | 4 +++- .../TypeConversion/DemoFix/op_kernel/demo.cpp | 2 -- 35 files changed, 148 insertions(+), 39 deletions(-) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp index eab8ce944..656b89476 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp @@ -76,10 +76,17 @@ private: uint32_t blockLength; uint32_t tileNum; uint32_t tileLength; - constexpr int OFFSET = 128; + constexpr int32_t OFFSET = 128; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelAdd op; diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp index b8e5524ce..60267eed1 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/add_custom.cpp @@ -74,10 +74,15 @@ private: GlobalTensor xGm; GlobalTensor yGm; GlobalTensor zGm; - constexpr int OFFSET = 128; + constexpr int32_t OFFSET = 128; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z) { KernelAdd op; op.Init(x, y, z); diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp index 4c21ef57c..19412362f 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp @@ -79,7 +79,14 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelAdd op; diff --git a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp index 4f377c719..0320148c0 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp @@ -76,7 +76,14 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelAdd op; diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp index 501ee0860..2ff8feccf 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_kernel/add_custom.cpp @@ -78,7 +78,14 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelAdd op; diff --git a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp +++ b/operator_contrib/CommonMistake/NoTilingKey/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp index ebd7a54bb..de39e95de 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp @@ -114,8 +114,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; diff --git a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100755 --- a/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index 70c169bc2..24dc15f86 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -8,10 +8,13 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(AddCustomCCTilingData) TILING_DATA_FIELD_DEF(uint32_t, totalLength); TILING_DATA_FIELD_DEF(uint32_t, tileNum); END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(AddCustom1, AddCustomCCTilingData) -} \ No newline at end of file +} +#endif // ADD_CUSTOM_TILING_H \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index 577408694..1741d15c2 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -78,7 +78,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom1(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) { +extern "C" +__global__ +__aicore__ +void add_custom1(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) +{ + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp index f432a3616..88a3aada1 100644 --- a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp @@ -76,8 +76,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp index 0c01e9c64..13276707a 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp @@ -26,7 +26,7 @@ static ge::graphStatus TilingFunc(gert::TilingContext *context) int64_t *ss = &shape[i * 3]; const gert::StorageShape* storageShape = context -> GetInputShape(i); for (int j = 0; j < storageShape->GetStorageShape().GetDimNum(); j++) { - constexpr int DIM_OFFSET = 2; + constexpr int32_t DIM_OFFSET = 2; ss[DIM_OFFSET * i + j] = storageShape->GetStorageShape().GetDim(j); } } diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp index a1efbf848..a1756c1ea 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp @@ -79,8 +79,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp index f5760c5cd..94d1e338a 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp @@ -79,8 +79,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100755 --- a/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp index 629577ca2..a2021c6c6 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp @@ -82,8 +82,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp index 384b915b7..79beec585 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp @@ -76,8 +76,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp index a1efbf848..a1756c1ea 100644 --- a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp @@ -79,8 +79,16 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h index c661fdbbd..fd5a7c2ec 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -8,10 +8,13 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) -} \ No newline at end of file +} +#endif // ADD_CUSTOM_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp index fe4be7923..61b20caae 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -26,7 +26,7 @@ public: } private: - // TODO: user kernel private member + __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::DataCopy(xLocal, xGm, this->totalLength); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h index c661fdbbd..fd5a7c2ec 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -8,10 +8,13 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) -} \ No newline at end of file +} +#endif // ADD_CUSTOM_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index a1d88fd64..02d3b73f9 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -26,7 +26,6 @@ public: } private: - // TODO: user kernel private member __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::DataCopy(xLocal, xGm, this->totalLength); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h index c661fdbbd..fd5a7c2ec 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -8,10 +8,13 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) -} \ No newline at end of file +} +#endif // ADD_CUSTOM_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index 16d1b421e..ca6b0ee84 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -28,7 +28,6 @@ public: } private: - // TODO: user kernel private member __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h index a1953af99..fd5a7c2ec 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -8,7 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" - +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -16,3 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } +#endif // ADD_CUSTOM_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index b0fe3954e..3b200086c 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -28,7 +28,6 @@ public: CopyOut(); } private: - // TODO: user kernel private member __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::LocalTensor yLocal = inQueueY.AllocTensor(); diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp index 47a39cd5f..ceaee0e4d 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/common.cpp @@ -70,7 +70,7 @@ bool WriteFile(const std::string &filePath, const void *buffer, size_t size) auto writeSize = write(fd, buffer, size); (void) close(fd); - if (writeSize != size) { + if (static_cast(writeSize) != static_cast(size)) { ERROR_LOG("Write file Failed."); return false; } diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h index a1953af99..fd5a7c2ec 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -8,7 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" - +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -16,3 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } +#endif // ADD_CUSTOM_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp index a2ab15a30..e44bea8e3 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -25,8 +25,6 @@ public: CopyOut(); } private: - // TODO: user kernel private member - __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::DataCopy(xLocal, xGm, this->totalLength); diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h index a1953af99..fd5a7c2ec 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -8,7 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" - +#ifndef ADD_CUSTOM_TILING_H +#define ADD_CUSTOM_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -16,3 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } +#endif // ADD_CUSTOM_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp index 16216d759..612580083 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -27,8 +27,6 @@ public: } private: - // TODO: user kernel private member - __aicore__ inline void CopyIn(){ AscendC::LocalTensor xLocal = inQueueX.AllocTensor(); AscendC::DataCopy(xLocal, xGm, this->totalLength); -- Gitee From 8d25c0bdf5bc23ae3d58b571ac1fabbf4b565c91 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Thu, 23 Jan 2025 09:39:38 +0800 Subject: [PATCH 07/17] update_little_error --- .../AclTensorFreeFail/AddKernelInvocation/main.cpp | 4 ++-- .../AddressMisalign/AddKernelInvocation/main.cpp | 4 ++-- .../NoTilingKey/AddCustom/op_kernel/add_custom.cpp | 1 - .../AddCustom/op_kernel/add_custom1.cpp | 1 - .../ResultError/AddCustom/op_kernel/add_custom.cpp | 12 +----------- .../AddCustom/op_host/add_custom.cpp | 2 +- .../AddCustom/op_kernel/add_custom.cpp | 14 ++------------ .../AddCustom/op_kernel/add_custom.cpp | 12 +----------- .../TimeOut/AddCustom/op_kernel/add_custom.cpp | 12 +----------- .../AddCustom/op_kernel/add_custom.cpp | 1 - .../AddCustom/op_kernel/add_custom.cpp | 12 +----------- .../DemoFix/op_kernel/demo.cpp | 1 - .../TypeConversion/DemoFix/op_kernel/demo.cpp | 4 +--- 13 files changed, 12 insertions(+), 68 deletions(-) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp index 3bfdff916..6cb6974f3 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddKernelInvocation/main.cpp @@ -19,8 +19,8 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z int32_t main(int32_t argc, char *argv[]) { uint32_t blockDim = 1; - size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); - size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t inputByteSize = static_cast(8) * static_cast(2048) * sizeof(uint16_t); + size_t outputByteSize = static_cast(8) * static_cast(2048) * sizeof(uint16_t); #ifdef ASCENDC_CPU_DEBUG uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp index 3bfdff916..6cb6974f3 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/main.cpp @@ -19,8 +19,8 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z int32_t main(int32_t argc, char *argv[]) { uint32_t blockDim = 1; - size_t inputByteSize = 8 * 2048 * sizeof(uint16_t); - size_t outputByteSize = 8 * 2048 * sizeof(uint16_t); + size_t inputByteSize = static_cast(8) * static_cast(2048) * sizeof(uint16_t); + size_t outputByteSize = static_cast(8) * static_cast(2048) * sizeof(uint16_t); #ifdef ASCENDC_CPU_DEBUG uint8_t *x = (uint8_t *)AscendC::GmAlloc(inputByteSize); diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp index de39e95de..518d02bb3 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_kernel/add_custom.cpp @@ -123,7 +123,6 @@ void add_custom(GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); KernelAdd op; diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index 1741d15c2..a776e50e8 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -87,7 +87,6 @@ void add_custom1(GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp index 88a3aada1..30cde3e7d 100644 --- a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp @@ -85,18 +85,8 @@ void add_custom(GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); op.Process(); -} - -#ifndef ASCENDC_CPU_DEBUG -// call of kernel function -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, - uint8_t *workspace, uint8_t *tiling) -{ - add_custom<<>>(x, y, z, workspace, tiling); -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp index 13276707a..729d3b768 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp @@ -20,7 +20,7 @@ static ge::graphStatus TilingFunc(gert::TilingContext *context) context->SetBlockDim(BLOCK_DIM); tiling.set_totalLength(totalLength); tiling.set_tileNum(TILE_NUM); - constexpr int INPUT_SHAPE_COUNT = 2; + constexpr int32_t INPUT_SHAPE_COUNT = 2; int64_t shape[6]; for (uint32_t i = 0; i < INPUT_SHAPE_COUNT; i++) { int64_t *ss = &shape[i * 3]; diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp index a1756c1ea..ce526515a 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_kernel/add_custom.cpp @@ -87,19 +87,9 @@ void add_custom(GM_ADDR x, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) -{ - +{ GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); op.Process(); -} - -#ifndef ASCENDC_CPU_DEBUG -// call of kernel function -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, - uint8_t *workspace, uint8_t *tiling) -{ - add_custom<<>>(x, y, z, workspace, tiling); -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp index 94d1e338a..5d700b875 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp @@ -88,18 +88,8 @@ void add_custom(GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); op.Process(); -} - -#ifndef ASCENDC_CPU_DEBUG -// call of kernel function -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, - uint8_t *workspace, uint8_t *tiling) -{ - add_custom<<>>(x, y, z, workspace, tiling); -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp index a2021c6c6..31e730db8 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp @@ -91,18 +91,8 @@ void add_custom(GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); op.Process(); -} -/* -#ifndef __CCE_KT_TEST__ -// call of kernel function -void add_custom_do(uint32_t blockDim, void* l2ctrl, void* stream, uint8_t* x, uint8_t* y, uint8_t* z, - uint8_t* workspace, uint8_t* tiling) -{ - add_custom<<>>(x, y, z, workspace, tiling); -} -#endif*/ +} \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp index 79beec585..438a9db5d 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp @@ -85,7 +85,6 @@ void add_custom(GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp index a1756c1ea..95ff9096b 100644 --- a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp @@ -88,18 +88,8 @@ void add_custom(GM_ADDR x, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); KernelAdd op; op.Init(x, y, z, tiling_data.totalLength, tiling_data.tileNum); op.Process(); -} - -#ifndef ASCENDC_CPU_DEBUG -// call of kernel function -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z, - uint8_t *workspace, uint8_t *tiling) -{ - add_custom<<>>(x, y, z, workspace, tiling); -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index 3b200086c..e1ccb2e8b 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -20,7 +20,6 @@ public: pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); pipe.InitBuffer(inQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); pipe.InitBuffer(outQueueZ, BUFFER_NUM, this->totalLength * sizeof(half)); - } __aicore__ inline void Process() { CopyIn(); diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp index 612580083..7b3038c6e 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -45,9 +45,7 @@ private: y +=x; sum =static_cast(y); yLocal.SetValue(i, sum); - } - - + } outQueueY.EnQue(yLocal); inQueueX.FreeTensor(xLocal); } -- Gitee From 1f6f30fbd4a9921bf402adaaf7184d7da3c6554a Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Thu, 23 Jan 2025 10:00:35 +0800 Subject: [PATCH 08/17] update_little_error --- .../InvalidMemory/AddCustom/op_kernel/add_custom.cpp | 11 +++++++---- .../NoTilingKey/AddCustom/op_host/add_custom.cpp | 10 +++++++++- .../tf_plugin/tensorflow_add_custom_cc_plugin.cc | 3 +-- .../OpNameError/AddCustom/op_host/add_custom_cc.cpp | 2 +- .../AddCustom/op_host/add_custom_cc_tiling.h | 2 +- .../OpNameError/AddCustom/op_kernel/add_custom1.cpp | 2 +- .../TimeOut/AddCustom/op_host/add_custom.cpp | 10 +++++++++- .../AclNNInvocation/src/op_runner.cpp | 5 +++-- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 3 +-- .../DifferentAlgorithm/Demo/op_host/demo.cpp | 2 +- .../DifferentAlgorithm/Demo/op_host/demo_tiling.h | 2 +- .../DifferentAlgorithm/Demo/op_kernel/demo.cpp | 2 +- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 3 +-- .../DifferentAlgorithm/DemoFix/op_host/demo.cpp | 2 +- .../DifferentAlgorithm/DemoFix/op_host/demo_tiling.h | 2 +- .../DifferentAlgorithm/DemoFix/op_kernel/demo.cpp | 2 +- .../AclNNInvocation/src/op_runner.cpp | 5 +++-- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 3 +-- .../SequenceOperationSteps/Demo/op_host/demo.cpp | 2 +- .../SequenceOperationSteps/Demo/op_host/demo_tiling.h | 2 +- .../SequenceOperationSteps/Demo/op_kernel/demo.cpp | 2 +- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 3 +-- .../SequenceOperationSteps/DemoFix/op_host/demo.cpp | 2 +- .../DemoFix/op_host/demo_tiling.h | 2 +- .../SequenceOperationSteps/DemoFix/op_kernel/demo.cpp | 2 +- .../TypeConversion/AclNNInvocation/src/op_runner.cpp | 5 +++-- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 3 +-- .../TypeConversion/Demo/op_host/demo.cpp | 2 +- .../TypeConversion/Demo/op_host/demo_tiling.h | 2 +- .../TypeConversion/Demo/op_kernel/demo.cpp | 2 +- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 3 +-- .../TypeConversion/DemoFix/op_host/demo.cpp | 2 +- .../TypeConversion/DemoFix/op_host/demo_tiling.h | 2 +- .../TypeConversion/DemoFix/op_kernel/demo.cpp | 2 +- 34 files changed, 62 insertions(+), 47 deletions(-) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp index 0320148c0..73b9e93a1 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp @@ -1,8 +1,11 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. +/** + * @file add_custom.cpp * - * Function : z = x + y - * This sample is a very basic sample that implements vector add on Ascend plaform. + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "kernel_operator.h" using namespace AscendC; diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp index 2ee711900..f36cad59f 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp @@ -1,4 +1,12 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "add_custom_tiling.h" #include "register/op_def_registry.h" #include "graph/utils/type_utils.h" diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc index 21064c663..1498fea59 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc @@ -1,5 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp index e46d31c79..36a98a2e0 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom_tiling.h + * @file add_custom_cc.h * * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index 24dc15f86..750f3c7b9 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -1,5 +1,5 @@ /** - * @file add_custom_tiling.h + * @file add_custom_tiling_cc.h * * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index a776e50e8..a63688436 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom_tiling.h + * @file add_custom1.cpp * * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp index 7c33bce2e..acb28e4c8 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp @@ -1,4 +1,12 @@ - +/** + * @file add_custom.cpp + * + * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "add_custom_tiling.h" #include "register/op_def_registry.h" #include "graph/utils/type_utils.h" diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp index 2934f4cae..d10e19ad5 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/AclNNInvocation/src/op_runner.cpp @@ -7,12 +7,13 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "op_runner.h" -#include "aclnn_demo.h" + #include #include #include "acl/acl_op_compiler.h" #include "common.h" +#include "op_runner.h" +#include "aclnn_demo.h" using namespace std; diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc index 6b6fc00c7..c4a341f06 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,5 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp index 4a69d0236..4854d568a 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h index fd5a7c2ec..6c4fc3cf8 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo_tiling.h * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp index 61b20caae..3d0f96b08 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc index 6b6fc00c7..c4a341f06 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,5 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp index 83c058814..e9958404d 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h index fd5a7c2ec..6c4fc3cf8 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo_tiling.h * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index 02d3b73f9..3ac8bd662 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp index 61911842a..e58e3e41f 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/AclNNInvocation/src/op_runner.cpp @@ -7,12 +7,13 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "op_runner.h" -#include "aclnn_demo.h" + #include #include #include "acl/acl_op_compiler.h" #include "common.h" +#include "op_runner.h" +#include "aclnn_demo.h" using namespace std; diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc index 6b6fc00c7..c4a341f06 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,5 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp index 5c51b8e30..fb149eaa0 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h index fd5a7c2ec..6c4fc3cf8 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo_tiling.h * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index ca6b0ee84..b60897999 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc index 6b6fc00c7..c4a341f06 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,5 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp index 3df8fa918..bc7e8e656 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h index fd5a7c2ec..6c4fc3cf8 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo_tiling.h * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index e1ccb2e8b..efd429a5b 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp index 2934f4cae..d10e19ad5 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/AclNNInvocation/src/op_runner.cpp @@ -7,12 +7,13 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "op_runner.h" -#include "aclnn_demo.h" + #include #include #include "acl/acl_op_compiler.h" #include "common.h" +#include "op_runner.h" +#include "aclnn_demo.h" using namespace std; diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc index 6b6fc00c7..c4a341f06 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,5 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp index 7c1a74d72..d439370ca 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h index fd5a7c2ec..6c4fc3cf8 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo_tiling.h * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp index e44bea8e3..9e113ab9a 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc index 6b6fc00c7..c4a341f06 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,5 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp index dce317aef..ce61a9346 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h index fd5a7c2ec..6c4fc3cf8 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo_tiling.h * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp index 7b3038c6e..7bb0a1344 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -1,5 +1,5 @@ /** - * @file add_custom.cpp + * @file demo.cpp * * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. * -- Gitee From 4673f014dce95782724603a2cc40aac92b24cc18 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Thu, 23 Jan 2025 10:20:55 +0800 Subject: [PATCH 09/17] update_many_error --- .../AclTensorFreeFail/run_test.sh | 12 ----------- .../CommonMistake/AddressMisalign/run_test.sh | 12 ----------- .../AddCustom/op_host/add_custom_cc_tiling.h | 6 +++--- .../SegmentationFault/run_test.sh | 12 ----------- .../AddKernelInvocation/add_custom.cpp | 3 ++- .../AddKernelInvocation/main.cpp | 8 ++++++-- .../CommonMistake/SynchronizeFail/run_test.sh | 12 ----------- .../TimeOut/AddCustom/op_host/add_custom.cpp | 2 +- .../CommonMistake/TypeDeletion/run_test.sh | 4 ---- .../Demo/op_host/demo_tiling.h | 6 +++--- .../DemoFix/op_host/demo_tiling.h | 6 +++--- .../DifferentAlgorithm/run_test.sh | 20 ------------------- .../Demo/op_host/demo_tiling.h | 6 +++--- .../DemoFix/op_host/demo_tiling.h | 6 +++--- .../SequenceOperationSteps/run_test.sh | 20 ------------------- .../TypeConversion/Demo/op_host/demo_tiling.h | 6 +++--- .../DemoFix/op_host/demo_tiling.h | 6 +++--- .../PrecisionTips/TypeConversion/run_test.sh | 20 ------------------- 18 files changed, 30 insertions(+), 137 deletions(-) delete mode 100755 operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh delete mode 100755 operator_contrib/CommonMistake/AddressMisalign/run_test.sh delete mode 100755 operator_contrib/CommonMistake/SegmentationFault/run_test.sh delete mode 100755 operator_contrib/CommonMistake/SynchronizeFail/run_test.sh delete mode 100755 operator_contrib/CommonMistake/TypeDeletion/run_test.sh delete mode 100755 operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh delete mode 100755 operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh delete mode 100755 operator_contrib/PrecisionTips/TypeConversion/run_test.sh diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh b/operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh deleted file mode 100755 index dde6963ce..000000000 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/run_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -LOG_PATH=/var/log/npu/slog/debug -if [ -d "$LOG_PATH" ]; then - echo "清理日志，路径：$LOG_PATH" -else - LOG_PATH=~/ascend/log - echo "清理日志，路径：$LOG_PATH" -fi -rm -rf $LOG_PATH* -cd AddCustom;./build.sh;./build_out/*.run -cd ../AclNNInvocation;bash run.sh -cd $LOG_PATH; grep -rn ERROR diff --git a/operator_contrib/CommonMistake/AddressMisalign/run_test.sh b/operator_contrib/CommonMistake/AddressMisalign/run_test.sh deleted file mode 100755 index e40ecfb5a..000000000 --- a/operator_contrib/CommonMistake/AddressMisalign/run_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -LOG_PATH=/var/log/npu/slog/debug -if [ -d "$LOG_PATH" ]; then - echo "清理日志，路径：$LOG_PATH" -else - LOG_PATH=~/ascend/log - echo "清理日志，路径：$LOG_PATH" -fi -rm -rf $LOG_PATH* -cd AddCustom;./build.sh;./build_out/*.run -cd ../AclNNInvocation;bash run.sh -cd $LOG_PATH; grep -rn errorStr \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index 750f3c7b9..b6584b2a1 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -8,8 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" -#ifndef ADD_CUSTOM_TILING_H -#define ADD_CUSTOM_TILING_H +#ifndef ADD_CUSTOM_CC_TILING_H +#define ADD_CUSTOM_CC_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(AddCustomCCTilingData) TILING_DATA_FIELD_DEF(uint32_t, totalLength); @@ -17,4 +17,4 @@ BEGIN_TILING_DATA_DEF(AddCustomCCTilingData) END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(AddCustom1, AddCustomCCTilingData) } -#endif // ADD_CUSTOM_TILING_H \ No newline at end of file +#endif // ADD_CUSTOM_CC_TILING_H \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SegmentationFault/run_test.sh b/operator_contrib/CommonMistake/SegmentationFault/run_test.sh deleted file mode 100755 index e40ecfb5a..000000000 --- a/operator_contrib/CommonMistake/SegmentationFault/run_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -LOG_PATH=/var/log/npu/slog/debug -if [ -d "$LOG_PATH" ]; then - echo "清理日志，路径：$LOG_PATH" -else - LOG_PATH=~/ascend/log - echo "清理日志，路径：$LOG_PATH" -fi -rm -rf $LOG_PATH* -cd AddCustom;./build.sh;./build_out/*.run -cd ../AclNNInvocation;bash run.sh -cd $LOG_PATH; grep -rn errorStr \ No newline at end of file diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp index bdf41a229..c247a304f 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp @@ -67,7 +67,8 @@ private: private: TPipe pipe; - TQue inQueueX, inQueueY; + TQue inQueueX; + TQue inQueueY; TQue outQueueZ; GlobalTensor xGm; GlobalTensor yGm; diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp index 75f01c7db..ead9970d4 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/main.cpp @@ -45,9 +45,13 @@ int32_t main(int32_t argc, char *argv[]) aclrtStream stream = nullptr; CHECK_ACL(aclrtCreateStream(&stream)); - uint8_t *xHost, *yHost, *zHost; - uint8_t *xDevice, *yDevice, *zDevice; + uint8_t *xHost; + uint8_t *yHost; + uint8_t *zHost; + uint8_t *xDevice; + uint8_t *yDevice; + uint8_t *zDevice; CHECK_ACL(aclrtMallocHost((void **)(&xHost), inputByteSize)); CHECK_ACL(aclrtMallocHost((void **)(&yHost), inputByteSize)); CHECK_ACL(aclrtMallocHost((void **)(&zHost), outputByteSize)); diff --git a/operator_contrib/CommonMistake/SynchronizeFail/run_test.sh b/operator_contrib/CommonMistake/SynchronizeFail/run_test.sh deleted file mode 100755 index 5facccd0a..000000000 --- a/operator_contrib/CommonMistake/SynchronizeFail/run_test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -LOG_PATH=/var/log/npu/slog/debug -if [ -d "$LOG_PATH" ]; then - echo "清理日志，路径：$LOG_PATH" -else - LOG_PATH=~/ascend/log - echo "清理日志，路径：$LOG_PATH" -fi -rm -rf $LOG_PATH* -cd AddCustom;./build.sh;./build_out/*.run -cd ../AclNNInvocation;bash run.sh -cd $LOG_PATH; grep -rn ERROR \ No newline at end of file diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp index acb28e4c8..cb5653a2a 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp @@ -25,7 +25,7 @@ static ge::graphStatus TilingFunc(gert::TilingContext* context) tiling.set_totalLength(totalLength); tiling.set_tileNum(TILE_NUM); - while(1); + while(1){}; tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); size_t *currentWorkspace = context->GetWorkspaceSizes(1); diff --git a/operator_contrib/CommonMistake/TypeDeletion/run_test.sh b/operator_contrib/CommonMistake/TypeDeletion/run_test.sh deleted file mode 100755 index 1de7c6039..000000000 --- a/operator_contrib/CommonMistake/TypeDeletion/run_test.sh +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf ~/ascend/log/* -cd AddCustom;./build.sh;./build_out/*.run -cd ../AclNNInvocation;bash run.sh -cd ~/ascend/log; grep -rn errorStr \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h index 6c4fc3cf8..f66cb2ec5 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -8,8 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" -#ifndef ADD_CUSTOM_TILING_H -#define ADD_CUSTOM_TILING_H +#ifndef DEMO_TILING_H +#define DEMO_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -17,4 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } -#endif // ADD_CUSTOM_TILING_H \ No newline at end of file +#endif // DEMO_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h index 6c4fc3cf8..f66cb2ec5 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -8,8 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" -#ifndef ADD_CUSTOM_TILING_H -#define ADD_CUSTOM_TILING_H +#ifndef DEMO_TILING_H +#define DEMO_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -17,4 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } -#endif // ADD_CUSTOM_TILING_H \ No newline at end of file +#endif // DEMO_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh b/operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh deleted file mode 100755 index 467a73309..000000000 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/run_test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -LOG_PATH=/var/log/npu/slog/debug -if [ -d "$LOG_PATH" ]; then - echo "清理日志，路径：$LOG_PATH" -else - LOG_PATH=~/ascend/log - echo "清理日志，路径：$LOG_PATH" -fi -rm -rf $LOG_PATH* - -current_path=$(pwd) -echo "*************** run Demo start****************" -cd Demo;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 -cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh -echo "*************** run Demo end****************" -cd $current_path -echo "*************** run DemoFix start****************" -cd DemoFix;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 -cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh -echo "*************** run DemoFix end****************" diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h index 6c4fc3cf8..f66cb2ec5 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -8,8 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" -#ifndef ADD_CUSTOM_TILING_H -#define ADD_CUSTOM_TILING_H +#ifndef DEMO_TILING_H +#define DEMO_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -17,4 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } -#endif // ADD_CUSTOM_TILING_H \ No newline at end of file +#endif // DEMO_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h index 6c4fc3cf8..f66cb2ec5 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -8,8 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" -#ifndef ADD_CUSTOM_TILING_H -#define ADD_CUSTOM_TILING_H +#ifndef DEMO_TILING_H +#define DEMO_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -17,4 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } -#endif // ADD_CUSTOM_TILING_H \ No newline at end of file +#endif // DEMO_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh b/operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh deleted file mode 100755 index 467a73309..000000000 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/run_test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -LOG_PATH=/var/log/npu/slog/debug -if [ -d "$LOG_PATH" ]; then - echo "清理日志，路径：$LOG_PATH" -else - LOG_PATH=~/ascend/log - echo "清理日志，路径：$LOG_PATH" -fi -rm -rf $LOG_PATH* - -current_path=$(pwd) -echo "*************** run Demo start****************" -cd Demo;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 -cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh -echo "*************** run Demo end****************" -cd $current_path -echo "*************** run DemoFix start****************" -cd DemoFix;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 -cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh -echo "*************** run DemoFix end****************" diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h index 6c4fc3cf8..f66cb2ec5 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -8,8 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" -#ifndef ADD_CUSTOM_TILING_H -#define ADD_CUSTOM_TILING_H +#ifndef DEMO_TILING_H +#define DEMO_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -17,4 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } -#endif // ADD_CUSTOM_TILING_H \ No newline at end of file +#endif // DEMO_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h index 6c4fc3cf8..f66cb2ec5 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -8,8 +8,8 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ #include "register/tilingdata_base.h" -#ifndef ADD_CUSTOM_TILING_H -#define ADD_CUSTOM_TILING_H +#ifndef DEMO_TILING_H +#define DEMO_TILING_H namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); @@ -17,4 +17,4 @@ END_TILING_DATA_DEF; REGISTER_TILING_DATA_CLASS(Demo, DemoTilingData) } -#endif // ADD_CUSTOM_TILING_H \ No newline at end of file +#endif // DEMO_TILING_H \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/TypeConversion/run_test.sh b/operator_contrib/PrecisionTips/TypeConversion/run_test.sh deleted file mode 100755 index 467a73309..000000000 --- a/operator_contrib/PrecisionTips/TypeConversion/run_test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -LOG_PATH=/var/log/npu/slog/debug -if [ -d "$LOG_PATH" ]; then - echo "清理日志，路径：$LOG_PATH" -else - LOG_PATH=~/ascend/log - echo "清理日志，路径：$LOG_PATH" -fi -rm -rf $LOG_PATH* - -current_path=$(pwd) -echo "*************** run Demo start****************" -cd Demo;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 -cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh -echo "*************** run Demo end****************" -cd $current_path -echo "*************** run DemoFix start****************" -cd DemoFix;./build.sh >/dev/null 2>&1;./build_out/*.run >/dev/null 2>&1 -cd ../AclNNInvocation >/dev/null 2>&1;bash run.sh -echo "*************** run DemoFix end****************" -- Gitee From 2f6c6d0c5699f7843d20e65497c9a095226ba756 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Thu, 23 Jan 2025 10:41:59 +0800 Subject: [PATCH 10/17] update_many_error --- .../AddKernelInvocation/add_custom.cpp | 14 ++++++++++++-- .../AddKernelInvocation/add_custom.cpp | 16 ++++++++++++++-- .../AddKernelInvocation/add_custom.cpp | 16 +++++++++++++--- .../AddCustom/op_host/add_custom_cc_tiling.h | 3 ++- .../AddKernelInvocation/add_custom.cpp | 16 +++++++++++++--- .../Demo/op_host/demo_tiling.h | 2 +- .../DifferentAlgorithm/Demo/op_kernel/demo.cpp | 8 +++++++- .../DemoFix/op_host/demo_tiling.h | 2 +- .../DemoFix/op_kernel/demo.cpp | 8 +++++++- .../Demo/op_host/demo_tiling.h | 2 +- .../Demo/op_kernel/demo.cpp | 10 ++++++++-- .../DemoFix/op_host/demo_tiling.h | 2 +- .../DemoFix/op_kernel/demo.cpp | 9 ++++++++- .../TypeConversion/Demo/op_host/demo_tiling.h | 2 +- .../TypeConversion/Demo/op_kernel/demo.cpp | 8 +++++++- .../TypeConversion/DemoFix/op_host/demo_tiling.h | 3 ++- .../TypeConversion/DemoFix/op_kernel/demo.cpp | 9 +++++++-- 17 files changed, 105 insertions(+), 25 deletions(-) diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp index 815ae452b..b35ebf408 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp @@ -76,7 +76,12 @@ private: GlobalTensor zGm; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z) { KernelAdd op; op.Init(x, y, z); @@ -85,7 +90,12 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z #ifndef ASCENDC_CPU_DEBUG // call of kernel function -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +void add_custom_do(uint32_t blockDim, + void *l2ctrl, + void *stream, + uint8_t *x, + uint8_t *y, + uint8_t *z) { add_custom<<>>(x, y, z); } diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp index 315d74dda..b1804142b 100644 --- a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp @@ -78,14 +78,26 @@ private: uint32_t tileLength; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { KernelAdd op; op.Init(x, y, z, 16384, 8); op.Process(); } #ifndef ASCENDC_CPU_DEBUG -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +void add_custom_do(uint32_t blockDim, + void *l2ctrl, + void *stream, + uint8_t *x, + uint8_t *y, + uint8_t *z) { add_custom<<>>(x, y, z); } diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp index b9906a3d8..07db99adf 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddKernelInvocation/add_custom.cpp @@ -75,7 +75,12 @@ private: GlobalTensor zGm; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z) { KernelAdd op; op.Init(x, y, z); @@ -84,8 +89,13 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z #ifndef ASCENDC_CPU_DEBUG // call of kernel function -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +void add_custom_do(uint32_t blockDim, + void *l2ctrl, + void *stream, + uint8_t *x, + uint8_t *y, + uint8_t *z) { add_custom<<>>(x, y, z); } -#endif +#endif \ No newline at end of file diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index b6584b2a1..4992d2db5 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -7,9 +7,10 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "register/tilingdata_base.h" + #ifndef ADD_CUSTOM_CC_TILING_H #define ADD_CUSTOM_CC_TILING_H +#include "register/tilingdata_base.h" namespace optiling { BEGIN_TILING_DATA_DEF(AddCustomCCTilingData) TILING_DATA_FIELD_DEF(uint32_t, totalLength); diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp index c247a304f..c34c50054 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddKernelInvocation/add_custom.cpp @@ -75,7 +75,12 @@ private: GlobalTensor zGm; }; -extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z) +extern "C" +__global__ +__aicore__ +void add_custom(GM_ADDR x, + GM_ADDR y, + GM_ADDR z) { KernelAdd op; op.Init(x, y, z); @@ -84,8 +89,13 @@ extern "C" __global__ __aicore__ void add_custom(GM_ADDR x, GM_ADDR y, GM_ADDR z #ifndef ASCENDC_CPU_DEBUG // call of kernel function -void add_custom_do(uint32_t blockDim, void *l2ctrl, void *stream, uint8_t *x, uint8_t *y, uint8_t *z) +void add_custom_do(uint32_t blockDim, + void *l2ctrl, + void *stream, + uint8_t *x, + uint8_t *y, + uint8_t *z) { add_custom<<>>(x, y, z); } -#endif +#endif \ No newline at end of file diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h index f66cb2ec5..4b56980e3 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -7,9 +7,9 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "register/tilingdata_base.h" #ifndef DEMO_TILING_H #define DEMO_TILING_H +#include "register/tilingdata_base.h" namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp index 3d0f96b08..17041d501 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -54,7 +54,13 @@ private: uint32_t totalLength; }; -extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { +extern "C" +__global__ +__aicore__ +void demo(GM_ADDR x, + GM_ADDR y, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; op.Init(x, y, tiling_data.size); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h index f66cb2ec5..4b56980e3 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -7,9 +7,9 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "register/tilingdata_base.h" #ifndef DEMO_TILING_H #define DEMO_TILING_H +#include "register/tilingdata_base.h" namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index 3ac8bd662..e60de5047 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -54,7 +54,13 @@ private: AscendC::GlobalTensor yGm; uint32_t totalLength; }; -extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { +extern "C" +__global__ +__aicore__ +void demo(GM_ADDR x, + GM_ADDR y, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; op.Init(x, y, tiling_data.size); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h index f66cb2ec5..4b56980e3 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -7,9 +7,9 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "register/tilingdata_base.h" #ifndef DEMO_TILING_H #define DEMO_TILING_H +#include "register/tilingdata_base.h" namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index b60897999..da2378165 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -64,8 +64,14 @@ private: AscendC::GlobalTensor zGm; uint32_t totalLength; }; - -extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) { +extern "C" +__global__ +__aicore__ +void demo(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; op.Init(x, y, z, tiling_data.size); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h index f66cb2ec5..4b56980e3 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -7,9 +7,9 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "register/tilingdata_base.h" #ifndef DEMO_TILING_H #define DEMO_TILING_H +#include "register/tilingdata_base.h" namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index efd429a5b..56ef8d908 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -64,7 +64,14 @@ private: AscendC::GlobalTensor zGm; uint32_t totalLength; }; -extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR z, GM_ADDR workspace, GM_ADDR tiling) { +extern "C" +__global__ +__aicore__ +void demo(GM_ADDR x, + GM_ADDR y, + GM_ADDR z, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; op.Init(x, y, z, tiling_data.size); diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h index f66cb2ec5..4b56980e3 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -7,9 +7,9 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "register/tilingdata_base.h" #ifndef DEMO_TILING_H #define DEMO_TILING_H +#include "register/tilingdata_base.h" namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp index 9e113ab9a..89dd765d6 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -61,7 +61,13 @@ private: AscendC::GlobalTensor yGm; uint32_t totalLength; }; -extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { +extern "C" +__global__ +__aicore__ +void demo(GM_ADDR x, + GM_ADDR y, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; op.Init(x, y, tiling_data.size); diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h index f66cb2ec5..d310f0d43 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -7,9 +7,10 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ -#include "register/tilingdata_base.h" + #ifndef DEMO_TILING_H #define DEMO_TILING_H +#include "register/tilingdata_base.h" namespace optiling { BEGIN_TILING_DATA_DEF(DemoTilingData) TILING_DATA_FIELD_DEF(uint32_t, size); diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp index 7bb0a1344..04ce44e66 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -18,7 +18,6 @@ public: yGm.SetGlobalBuffer((__gm__ half *)y, this->totalLength); pipe.InitBuffer(inQueueX, BUFFER_NUM, this->totalLength * sizeof(half)); pipe.InitBuffer(outQueueY, BUFFER_NUM, this->totalLength * sizeof(half)); - } __aicore__ inline void Process() { CopyIn(); @@ -63,7 +62,13 @@ private: uint32_t totalLength; }; -extern "C" __global__ __aicore__ void demo(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { +extern "C" +__global__ +__aicore__ +void demo(GM_ADDR x, + GM_ADDR y, + GM_ADDR workspace, + GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); KernelDemo op; op.Init(x, y, tiling_data.size); -- Gitee From 402174676087a527a05200f0821ed5aef11809c7 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Thu, 23 Jan 2025 11:08:42 +0800 Subject: [PATCH 11/17] update_copyright --- .../AclTensorFreeFail/AddCustom/op_host/add_custom.cpp | 2 +- .../AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h | 2 +- .../AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp | 2 +- .../AddressMisalign/AddCustom/op_host/add_custom.cpp | 2 +- .../AddressMisalign/AddCustom/op_host/add_custom_tiling.h | 2 +- .../AddressMisalign/AddCustom/op_kernel/add_custom.cpp | 2 +- .../AddressMisalign/AddKernelInvocation/add_custom.cpp | 2 +- .../InvalidMemory/AddCustom/op_host/add_custom_tiling.h | 2 +- .../InvalidMemory/AddCustom/op_kernel/add_custom.cpp | 2 +- .../InvalidMemory/AddKernelInvocation/add_custom.cpp | 4 +++- .../MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp | 2 +- .../AddCustom/op_host/add_custom_tiling.h | 2 +- .../NoTilingKey/AddCustom/op_host/add_custom.cpp | 2 +- .../NoTilingKey/AddCustom/op_host/add_custom_tiling.h | 2 +- .../OpNameError/AddCustom/op_host/add_custom_cc.cpp | 2 +- .../OpNameError/AddCustom/op_host/add_custom_cc_tiling.h | 2 +- .../OpNameError/AddCustom/op_kernel/add_custom1.cpp | 2 +- .../ResultError/AddCustom/op_host/add_custom.cpp | 2 +- .../ResultError/AddCustom/op_host/add_custom_tiling.h | 2 +- .../ResultError/AddCustom/op_kernel/add_custom.cpp | 2 +- .../SegmentationFault/AddCustom/op_host/add_custom.cpp | 2 +- .../SegmentationFault/AddCustom/op_host/add_custom_tiling.h | 2 +- .../SynchronizeFail/AddCustom/op_host/add_custom.cpp | 2 +- .../SynchronizeFail/AddCustom/op_host/add_custom_tiling.h | 2 +- .../SynchronizeFail/AddCustom/op_kernel/add_custom.cpp | 2 +- .../CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp | 2 +- .../TimeOut/AddCustom/op_host/add_custom_tiling.h | 2 +- .../CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp | 2 +- .../TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp | 2 +- .../TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h | 2 +- .../TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp | 2 +- .../TypeDeletion/AddCustom/op_host/add_custom.cpp | 2 +- .../TypeDeletion/AddCustom/op_host/add_custom_tiling.h | 2 +- .../TypeDeletion/AddCustom/op_kernel/add_custom.cpp | 2 +- 34 files changed, 36 insertions(+), 34 deletions(-) diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp index 6fc77c881..c63bb137b 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h index 323f3076f..c9444ff30 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp index 656b89476..3f803f994 100644 --- a/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/AclTensorFreeFail/AddCustom/op_kernel/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2025. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp index 6fc77c881..c63bb137b 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h index 323f3076f..c9444ff30 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp index 19412362f..c86564601 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddCustom/op_kernel/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp index b35ebf408..467ad5c86 100644 --- a/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/AddressMisalign/AddKernelInvocation/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h index 4bb6d940f..f41e97ead 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom_tiling.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. */ #ifndef ADD_CUSTOM_TILING_H #define ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp index 73b9e93a1..975db5679 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_kernel/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp index b1804142b..f3e528040 100644 --- a/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddKernelInvocation/add_custom.cpp @@ -9,6 +9,8 @@ using namespace AscendC; constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue constexpr int32_t BUFFER_SIZE = 4000; constexpr int32_t DATA_SIZE = 2048; +constexpr int32_t ALL_SIZE = 16384; +constexpr int32_t ALL_CORE = 8; constexpr int32_t OVER_SIZE = BUFFER_SIZE*BUFFER_SIZE*8; class KernelAdd { public: @@ -88,7 +90,7 @@ void add_custom(GM_ADDR x, GM_ADDR tiling) { KernelAdd op; - op.Init(x, y, z, 16384, 8); + op.Init(x, y, z, ALL_SIZE, ALL_CORE); op.Process(); } #ifndef ASCENDC_CPU_DEBUG diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp index 3a6671209..081370f3b 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h index 323f3076f..c9444ff30 100644 --- a/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/MPUAddressAccessInvalid/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp index f36cad59f..0f6fa13cc 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h index 4bb6d940f..f41e97ead 100755 --- a/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/NoTilingKey/AddCustom/op_host/add_custom_tiling.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. */ #ifndef ADD_CUSTOM_TILING_H #define ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp index 36a98a2e0..f194b823e 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp @@ -1,7 +1,7 @@ /** * @file add_custom_cc.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index 4992d2db5..f8aa87b36 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling_cc.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index a63688436..587f90696 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -1,7 +1,7 @@ /** * @file add_custom1.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp index 3a6671209..081370f3b 100644 --- a/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h index 323f3076f..c9444ff30 100644 --- a/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp index 30cde3e7d..1cb584bd3 100644 --- a/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/ResultError/AddCustom/op_kernel/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp index 729d3b768..17e469f0c 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h index 5f7f213a3..a60aa0e58 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/SegmentationFault/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp index 6fc77c881..c63bb137b 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h index 323f3076f..c9444ff30 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp index 5d700b875..e398fc471 100644 --- a/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/SynchronizeFail/AddCustom/op_kernel/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp index cb5653a2a..3ffb0c323 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h index 4bb6d940f..f41e97ead 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_host/add_custom_tiling.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. */ #ifndef ADD_CUSTOM_TILING_H #define ADD_CUSTOM_TILING_H diff --git a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp index 31e730db8..55b4f8a95 100755 --- a/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOut/AddCustom/op_kernel/add_custom.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * Function : z = x + y * This sample is a very basic sample that implements vector add on Ascend plaform. diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp index 6fc77c881..c63bb137b 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h index 323f3076f..c9444ff30 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp index 438a9db5d..94c154060 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AddCustom/op_kernel/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp index 6fc77c881..c63bb137b 100644 --- a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h index 323f3076f..c9444ff30 100644 --- a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_host/add_custom_tiling.h @@ -1,7 +1,7 @@ /** * @file add_custom_tiling.h * - * Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp index 95ff9096b..5ba62279d 100644 --- a/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp +++ b/operator_contrib/CommonMistake/TypeDeletion/AddCustom/op_kernel/add_custom.cpp @@ -1,7 +1,7 @@ /** * @file add_custom.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of -- Gitee From abe13a431c7032c9842868193319078efcd59477 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Thu, 23 Jan 2025 11:09:11 +0800 Subject: [PATCH 12/17] update_copyright --- .../PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp | 2 +- .../DifferentAlgorithm/Demo/op_host/demo_tiling.h | 2 +- .../DifferentAlgorithm/DemoFix/op_host/demo.cpp | 5 +++-- .../DifferentAlgorithm/DemoFix/op_host/demo_tiling.h | 2 +- .../SequenceOperationSteps/Demo/op_host/demo.cpp | 2 +- .../SequenceOperationSteps/Demo/op_host/demo_tiling.h | 2 +- .../SequenceOperationSteps/Demo/op_kernel/demo.cpp | 2 +- .../SequenceOperationSteps/DemoFix/op_host/demo.cpp | 2 +- .../SequenceOperationSteps/DemoFix/op_host/demo_tiling.h | 2 +- .../SequenceOperationSteps/DemoFix/op_kernel/demo.cpp | 2 +- .../PrecisionTips/TypeConversion/Demo/op_host/demo.cpp | 2 +- .../PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h | 2 +- .../PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp | 2 +- .../PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp | 2 +- .../TypeConversion/DemoFix/op_host/demo_tiling.h | 2 +- .../PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp | 2 +- 16 files changed, 18 insertions(+), 17 deletions(-) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp index 4854d568a..8d9d684ed 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h index 4b56980e3..8ea093935 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -1,7 +1,7 @@ /** * @file demo_tiling.h * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp index e9958404d..3d174feba 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp @@ -1,13 +1,14 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * - /#include "demo_tiling.h" + */ + #include "demo_tiling.h" #include "register/op_def_registry.h" namespace optiling { static ge::graphStatus TilingFunc(gert::TilingContext* context) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h index 4b56980e3..8ea093935 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -1,7 +1,7 @@ /** * @file demo_tiling.h * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp index fb149eaa0..b35756c1d 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h index 4b56980e3..8ea093935 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -1,7 +1,7 @@ /** * @file demo_tiling.h * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index da2378165..9ec04027f 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp index bc7e8e656..0ba0e5302 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h index 4b56980e3..8ea093935 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -1,7 +1,7 @@ /** * @file demo_tiling.h * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index 56ef8d908..c5a359e08 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp index d439370ca..786629922 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h index 4b56980e3..8ea093935 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -1,7 +1,7 @@ /** * @file demo_tiling.h * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp index 89dd765d6..42795bf68 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp index ce61a9346..12c2e527a 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h index d310f0d43..8216de455 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -1,7 +1,7 @@ /** * @file demo_tiling.h * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp index 04ce44e66..5b5331ebe 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of -- Gitee From 4d1968025e239c704e7f9b882aab97429b03765a Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Thu, 23 Jan 2025 11:22:55 +0800 Subject: [PATCH 13/17] update_copyright --- .../InvalidMemory/AddCustom/op_host/add_custom.cpp | 10 +++++++++- .../tf_plugin/tensorflow_add_custom_cc_plugin.cc | 2 +- .../Demo/framework/tf_plugin/tensorflow_demo_plugin.cc | 2 +- .../DifferentAlgorithm/Demo/op_kernel/demo.cpp | 2 +- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 2 +- .../DifferentAlgorithm/DemoFix/op_kernel/demo.cpp | 2 +- .../Demo/framework/tf_plugin/tensorflow_demo_plugin.cc | 2 +- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 2 +- .../Demo/framework/tf_plugin/tensorflow_demo_plugin.cc | 2 +- .../framework/tf_plugin/tensorflow_demo_plugin.cc | 2 +- 10 files changed, 18 insertions(+), 10 deletions(-) diff --git a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp index 3262f2c31..7558839d2 100755 --- a/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp +++ b/operator_contrib/CommonMistake/InvalidMemory/AddCustom/op_host/add_custom.cpp @@ -1,4 +1,12 @@ - +/** + * @file add_custom.cpp + * + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "add_custom_tiling.h" #include "register/op_def_registry.h" #include "graph/utils/type_utils.h" diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc index 1498fea59..adb6bf06e 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/framework/tf_plugin/tensorflow_add_custom_cc_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. +/* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc index c4a341f06..5429b5108 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. +/* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp index 17041d501..005e9ab03 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + *Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc index c4a341f06..5429b5108 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. +/* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index e60de5047..f7553be6c 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -1,7 +1,7 @@ /** * @file demo.cpp * - * Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. + *Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc index c4a341f06..5429b5108 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. +/* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc index c4a341f06..5429b5108 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. +/* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc index c4a341f06..5429b5108 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. +/* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc index c4a341f06..5429b5108 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/framework/tf_plugin/tensorflow_demo_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved. +/* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the Apache License Version 2.0. -- Gitee From 00060f77c8aa5e17c3b5451996a9d0b50eeb9ad7 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Fri, 24 Jan 2025 11:22:39 +0800 Subject: [PATCH 14/17] update_copyright --- .../OpNameError/AddCustom/op_host/add_custom_cc.cpp | 10 +++++----- .../AddCustom/op_host/add_custom_cc_tiling.h | 11 +++++------ .../OpNameError/AddCustom/op_kernel/add_custom1.cpp | 10 +++++----- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp index f194b823e..eed83a95b 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp @@ -1,12 +1,12 @@ -/** - * @file add_custom_cc.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ +/** + * @file add_custom_cc.cpp + */ #include "add_custom_cc_tiling.h" #include "register/op_def_registry.h" namespace optiling { diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index f8aa87b36..a22728038 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -1,13 +1,12 @@ -/** - * @file add_custom_tiling_cc.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ - +/** + * @file add_custom_cc_tiling.h + */ #ifndef ADD_CUSTOM_CC_TILING_H #define ADD_CUSTOM_CC_TILING_H #include "register/tilingdata_base.h" diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index 587f90696..b89215af4 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -1,12 +1,12 @@ -/** - * @file add_custom1.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. */ +/** + * @file add_custom1.cpp + */ #include "kernel_operator.h" using namespace AscendC; constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue -- Gitee From b102607d36bbe645b87d39bb759f38ee3914a3a9 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Fri, 24 Jan 2025 11:37:44 +0800 Subject: [PATCH 15/17] update_copyright --- .../OpNameError/AddCustom/op_host/add_custom_cc.cpp | 13 ++++++++++++- .../AddCustom/op_host/add_custom_cc_tiling.h | 13 ++++++++++++- .../OpNameError/AddCustom/op_kernel/add_custom1.cpp | 13 ++++++++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp index eed83a95b..8437f647e 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc.cpp @@ -3,8 +3,19 @@ * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -/** + /** * @file add_custom_cc.cpp */ #include "add_custom_cc_tiling.h" diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h index a22728038..c56f11455 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_host/add_custom_cc_tiling.h @@ -3,8 +3,19 @@ * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -/** + /** * @file add_custom_cc_tiling.h */ #ifndef ADD_CUSTOM_CC_TILING_H diff --git a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp index b89215af4..c98e1d1ca 100755 --- a/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp +++ b/operator_contrib/CommonMistake/OpNameError/AddCustom/op_kernel/add_custom1.cpp @@ -3,8 +3,19 @@ * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -/** + /** * @file add_custom1.cpp */ #include "kernel_operator.h" -- Gitee From 8074b92de46e7cca5fd37233bd9952df0c59cc69 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Fri, 24 Jan 2025 14:06:18 +0800 Subject: [PATCH 16/17] update_copyright --- .../DifferentAlgorithm/Demo/op_host/demo.cpp | 21 +++++++++++++----- .../Demo/op_host/demo_tiling.h | 21 +++++++++++++----- .../Demo/op_kernel/demo.cpp | 21 +++++++++++++----- .../DemoFix/op_host/demo.cpp | 20 ++++++++++++----- .../DemoFix/op_host/demo_tiling.h | 21 +++++++++++++----- .../DemoFix/op_kernel/demo.cpp | 21 +++++++++++++----- .../Demo/op_host/demo.cpp | 21 +++++++++++++----- .../Demo/op_host/demo_tiling.h | 21 +++++++++++++----- .../Demo/op_kernel/demo.cpp | 21 +++++++++++++----- .../DemoFix/op_host/demo.cpp | 21 +++++++++++++----- .../DemoFix/op_host/demo_tiling.h | 21 +++++++++++++----- .../DemoFix/op_kernel/demo.cpp | 21 +++++++++++++----- .../TypeConversion/Demo/op_host/demo.cpp | 21 +++++++++++++----- .../TypeConversion/Demo/op_host/demo_tiling.h | 21 +++++++++++++----- .../TypeConversion/Demo/op_kernel/demo.cpp | 21 +++++++++++++----- .../TypeConversion/DemoFix/op_host/demo.cpp | 21 +++++++++++++----- .../DemoFix/op_host/demo_tiling.h | 22 ++++++++++++++----- .../TypeConversion/DemoFix/op_kernel/demo.cpp | 21 +++++++++++++----- 18 files changed, 287 insertions(+), 91 deletions(-) diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp index 8d9d684ed..2bd82288a 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "demo_tiling.h" #include "register/op_def_registry.h" diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h index 8ea093935..c533dee8e 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_host/demo_tiling.h @@ -1,11 +1,22 @@ -/** - * @file demo_tiling.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo_tiling.h */ #ifndef DEMO_TILING_H #define DEMO_TILING_H diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp index 005e9ab03..253dd993a 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/Demo/op_kernel/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - *Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp index 3d174feba..f04138563 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo.cpp @@ -1,12 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "demo_tiling.h" #include "register/op_def_registry.h" diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h index 8ea093935..c533dee8e 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_host/demo_tiling.h @@ -1,11 +1,22 @@ -/** - * @file demo_tiling.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo_tiling.h */ #ifndef DEMO_TILING_H #define DEMO_TILING_H diff --git a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp index f7553be6c..cc365a579 100644 --- a/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/DifferentAlgorithm/DemoFix/op_kernel/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - *Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp index b35756c1d..72a46532e 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "demo_tiling.h" #include "register/op_def_registry.h" diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h index 8ea093935..c533dee8e 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_host/demo_tiling.h @@ -1,11 +1,22 @@ -/** - * @file demo_tiling.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo_tiling.h */ #ifndef DEMO_TILING_H #define DEMO_TILING_H diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp index 9ec04027f..d35435d99 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/Demo/op_kernel/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp index 0ba0e5302..674f6340e 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "demo_tiling.h" #include "register/op_def_registry.h" diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h index 8ea093935..c533dee8e 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_host/demo_tiling.h @@ -1,11 +1,22 @@ -/** - * @file demo_tiling.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo_tiling.h */ #ifndef DEMO_TILING_H #define DEMO_TILING_H diff --git a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp index c5a359e08..6e16576b5 100644 --- a/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/SequenceOperationSteps/DemoFix/op_kernel/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp index 786629922..96477c679 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "demo_tiling.h" #include "register/op_def_registry.h" diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h index 8ea093935..c533dee8e 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_host/demo_tiling.h @@ -1,11 +1,22 @@ -/** - * @file demo_tiling.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo_tiling.h */ #ifndef DEMO_TILING_H #define DEMO_TILING_H diff --git a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp index 42795bf68..4e00bac7e 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/Demo/op_kernel/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp index 12c2e527a..f8c88334e 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "demo_tiling.h" #include "register/op_def_registry.h" diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h index 8216de455..c533dee8e 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_host/demo_tiling.h @@ -1,13 +1,23 @@ -/** - * @file demo_tiling.h - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo_tiling.h */ - #ifndef DEMO_TILING_H #define DEMO_TILING_H #include "register/tilingdata_base.h" diff --git a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp index 5b5331ebe..a329c2a7b 100644 --- a/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp +++ b/operator_contrib/PrecisionTips/TypeConversion/DemoFix/op_kernel/demo.cpp @@ -1,11 +1,22 @@ -/** - * @file demo.cpp - * - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** + * @file demo.cpp */ #include "kernel_operator.h" constexpr int32_t BUFFER_NUM = 1; // tensor num for each queue -- Gitee From 8bd3d6407f3644b826df02a8894972d675ae5f55 Mon Sep 17 00:00:00 2001 From: Ch_confuse <1532701447@qq.com> Date: Fri, 24 Jan 2025 14:16:02 +0800 Subject: [PATCH 17/17] update_copyright --- .../CommonMistake/SegmentationFault/tiling_debug.py | 13 ++++++++++++- .../AclNNInvocation/scripts/get_time.py | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py b/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py index 86d6f3c61..46ab23593 100644 --- a/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py +++ b/operator_contrib/CommonMistake/SegmentationFault/tiling_debug.py @@ -1,11 +1,22 @@ #!/usr/bin/python3 # coding=utf-8 # -# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # =============================================================================== import ascendebug import numpy as np diff --git a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py index b6214ef2d..356f9b3c4 100644 --- a/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py +++ b/operator_contrib/CommonMistake/TimeOverrunBaseline/AclNNInvocation/scripts/get_time.py @@ -1,11 +1,22 @@ #!/usr/bin/python3 # coding=utf-8 # -# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Copyright (C) Huawei Technologies Co., Ltd. 2025. All rights reserved. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # =============================================================================== import csv from pathlib import Path -- Gitee