1 Star 0 Fork 1.1K

peng/pytorch

forked from Ascend/pytorch 
加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
setup.py 11.29 KB
一键复制 编辑 原始数据 按行查看 历史
尤安升 提交于 2022-04-16 15:39 +08:00 . Delete author info.
# Copyright (c) 2020 Huawei Technologies Co., Ltd
# All rights reserved.
#
# Licensed under the BSD 3-Clause License (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import glob
import multiprocessing
import multiprocessing.pool
import os
import re
import shutil
import subprocess
import sys
import site
import platform
from sysconfig import get_paths
import distutils.ccompiler
import distutils.command.clean
from distutils.version import LooseVersion
from distutils.command.build_py import build_py
from setuptools.command.build_ext import build_ext
from setuptools.command.install import install
from setuptools import setup, distutils, Extension
from setuptools.command.build_clib import build_clib
from setuptools.command.egg_info import egg_info
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
VERSION = '1.8.1rc1'
def which(thefile):
path = os.environ.get("PATH", os.defpath).split(os.pathsep)
for d in path:
fname = os.path.join(d, thefile)
fnames = [fname]
if sys.platform == 'win32':
exts = os.environ.get('PATHEXT', '').split(os.pathsep)
fnames += [fname + ext for ext in exts]
for name in fnames:
if os.access(name, os.F_OK | os.X_OK) and not os.path.isdir(name):
return name
return None
def get_cmake_command():
def _get_version(cmd):
for line in subprocess.check_output([cmd, '--version']).decode('utf-8').split('\n'):
if 'version' in line:
return LooseVersion(line.strip().split(' ')[2])
raise RuntimeError('no version found')
"Returns cmake command."
cmake_command = 'cmake'
if platform.system() == 'Windows':
return cmake_command
cmake3 = which('cmake3')
cmake = which('cmake')
if cmake3 is not None and _get_version(cmake3) >= LooseVersion("3.13.0"):
cmake_command = 'cmake3'
return cmake_command
elif cmake is not None and _get_version(cmake) >= LooseVersion("3.13.0"):
return cmake_command
else:
raise RuntimeError('no cmake or cmake3 with version >= 3.13.0 found')
def get_build_type():
build_type = 'Release'
if os.getenv('DEBUG', default='0').upper() in ['ON', '1', 'YES', 'TRUE', 'Y']:
build_type = 'Debug'
if os.getenv('REL_WITH_DEB_INFO', default='0').upper() in ['ON', '1', 'YES', 'TRUE', 'Y']:
build_type = 'RelWithDebInfo'
return build_type
def _get_build_mode():
for i in range(1, len(sys.argv)):
if not sys.argv[i].startswith('-'):
return sys.argv[i]
def get_package_dir():
if '--user' in sys.argv:
package_dir = site.getusersitepackages()
else:
py_version = f'{sys.version_info.major}.{sys.version_info.minor}'
package_dir = f'{sys.prefix}/lib/python{py_version}/site-packages'
return package_dir
def generate_bindings_code(base_dir):
generate_code_cmd = ["sh", os.path.join(base_dir, 'scripts', 'generate_code.sh')]
if subprocess.call(generate_code_cmd) != 0:
print(
'Failed to generate ATEN bindings: {}'.format(generate_code_cmd),
file=sys.stderr)
sys.exit(1)
def build_stub(base_dir):
build_stub_cmd = ["sh", os.path.join(base_dir, 'third_party/acl/libs/build_stub.sh')]
if subprocess.call(build_stub_cmd) != 0:
print(
'Failed to build stub: {}'.format(build_stub_cmd),
file=sys.stderr)
sys.exit(1)
def CppExtension(name, sources, *args, **kwargs):
r'''
Creates a :class:`setuptools.Extension` for C++.
'''
package_dir = get_package_dir()
temp_include_dirs = kwargs.get('include_dirs', [])
temp_include_dirs.append(os.path.join(package_dir, 'torch/include'))
temp_include_dirs.append(os.path.join(package_dir, 'torch/include/torch/csrc/api/include'))
kwargs['include_dirs'] = temp_include_dirs
temp_library_dirs = kwargs.get('library_dirs', [])
temp_library_dirs.append(os.path.join(package_dir, 'torch/lib'))
temp_library_dirs.append(os.path.join(BASE_DIR, "third_party/acl/libs"))
kwargs['library_dirs'] = temp_library_dirs
libraries = kwargs.get('libraries', [])
libraries.append('c10')
libraries.append('torch')
libraries.append('torch_cpu')
libraries.append('torch_python')
libraries.append('hccl')
kwargs['libraries'] = libraries
kwargs['language'] = 'c++'
return Extension(name, sources, *args, **kwargs)
class Clean(distutils.command.clean.clean):
def run(self):
f_ignore = open('.gitignore', 'r')
ignores = f_ignore.read()
pat = re.compile(r'^#( BEGIN NOT-CLEAN-FILES )?')
for wildcard in filter(None, ignores.split('\n')):
match = pat.match(wildcard)
if match:
if match.group(1):
# Marker is found and stop reading .gitignore.
break
# Ignore lines which begin with '#'.
else:
for filename in glob.glob(wildcard):
shutil.rmtree(filename, ignore_errors=True)
f_ignore.close()
# It's an old-style class in Python 2.7...
distutils.command.clean.clean.run(self)
class CPPLibBuild(build_clib, object):
def run(self):
cmake = get_cmake_command()
if cmake is None:
raise RuntimeError(
"CMake must be installed to build the following extensions: " +
", ".join(e.name for e in self.extensions))
self.cmake = cmake
build_dir = os.path.join(BASE_DIR, "build")
build_type_dir = os.path.join(build_dir, get_build_type())
output_lib_path = os.path.join(build_type_dir, "packages/torch_npu/lib")
os.makedirs(build_type_dir, exist_ok=True)
os.makedirs(output_lib_path, exist_ok=True)
self.build_lib = os.path.relpath(os.path.join(build_dir, "packages/torch_npu"))
self.build_temp = os.path.relpath(build_type_dir)
cmake_args = [
'-DCMAKE_BUILD_TYPE=' + get_build_type(),
'-DCMAKE_INSTALL_PREFIX=' + os.path.abspath(output_lib_path),
'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + os.path.abspath(output_lib_path),
'-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=' + os.path.abspath(output_lib_path),
'-DTORCHNPU_INSTALL_LIBDIR=' + os.path.abspath(output_lib_path),
'-DPYTHON_INCLUDE_DIR=' + get_paths()['include'],
'-DPYTORCH_INSTALL_DIR=' + os.path.join(get_package_dir(), "torch")]
build_args = ['-j', str(multiprocessing.cpu_count())]
subprocess.check_call([self.cmake, BASE_DIR] + cmake_args, cwd=build_type_dir, env=os.environ)
subprocess.check_call(['make'] + build_args, cwd=build_type_dir, env=os.environ)
class Build(build_ext, object):
def run(self):
self.run_command('build_clib')
self.build_lib = os.path.relpath(os.path.join(BASE_DIR, f"build/{get_build_type()}/packages"))
self.build_temp = os.path.relpath(os.path.join(BASE_DIR, f"build/{get_build_type()}"))
self.library_dirs.append(
os.path.relpath(os.path.join(BASE_DIR, f"build/{get_build_type()}/packages/torch_npu/lib")))
super(Build, self).run()
class InstallCmd(install):
def finalize_options(self) -> None:
self.build_lib = os.path.relpath(os.path.join(BASE_DIR, f"build/{get_build_type()}/packages"))
return super(InstallCmd, self).finalize_options()
def get_src_py_and_dst():
ret = []
generated_python_files = glob.glob(
os.path.join(BASE_DIR, "torch_npu", '**/*.py'),
recursive=True)
for src in generated_python_files:
dst = os.path.join(
os.path.join(BASE_DIR, f"build/{get_build_type()}/packages/torch_npu"),
os.path.relpath(src, os.path.join(BASE_DIR, "torch_npu")))
os.makedirs(os.path.dirname(dst), exist_ok=True)
ret.append((src, dst))
return ret
class EggInfoBuild(egg_info, object):
def finalize_options(self):
self.egg_base = os.path.relpath(os.path.join(BASE_DIR, f"build/{get_build_type()}/packages"))
ret = get_src_py_and_dst()
for src, dst in ret:
self.copy_file(src, dst)
super(EggInfoBuild, self).finalize_options()
class PythonPackageBuild(build_py, object):
def run(self) -> None:
ret = get_src_py_and_dst()
for src, dst in ret:
self.copy_file(src, dst)
super(PythonPackageBuild, self).finalize_options()
build_mode = _get_build_mode()
if build_mode not in ['clean']:
# Generate bindings code, including RegisterNPU.cpp & NPUNativeFunctions.h.
generate_bindings_code(BASE_DIR)
build_stub(BASE_DIR)
# Setup include directories folders.
include_directories = [
BASE_DIR,
os.path.join(BASE_DIR, 'torch_npu/csrc/aten'),
os.path.join(BASE_DIR, 'third_party/hccl/inc'),
os.path.join(BASE_DIR, 'third_party/acl/inc')
]
extra_link_args = []
DEBUG = (os.getenv('DEBUG', default='').upper() in ['ON', '1', 'YES', 'TRUE', 'Y'])
extra_compile_args = [
'-std=c++14',
'-Wno-sign-compare',
'-Wno-deprecated-declarations',
'-Wno-return-type',
]
if re.match(r'clang', os.getenv('CC', '')):
extra_compile_args += [
'-Wno-macro-redefined',
'-Wno-return-std-move',
]
if DEBUG:
extra_compile_args += ['-O0', '-g']
extra_link_args += ['-O0', '-g', '-Wl,-z,now']
else:
extra_compile_args += ['-DNDEBUG']
extra_link_args += ['-Wl,-z,now,-s']
setup(
name=os.environ.get('TORCH_NPU_PACKAGE_NAME', 'torch_npu'),
version=VERSION,
description='NPU bridge for PyTorch',
url='https://gitee.com/ascend/pytorch',
packages=["torch_npu"],
libraries=[('torch_npu', {'sources': list()})],
package_dir={'': os.path.relpath(os.path.join(BASE_DIR, f"build/{get_build_type()}/packages"))},
ext_modules=[
CppExtension(
'torch_npu._C',
sources=["torch_npu/csrc/InitNpuBindings.cpp"],
libraries=["torch_npu"],
include_dirs=include_directories,
extra_compile_args=extra_compile_args + ['-fstack-protector-all'],
library_dirs=["lib"],
extra_link_args=extra_link_args + ['-Wl,-rpath,$ORIGIN/lib'],
),
],
extras_require={
},
package_data={
'torch_npu': [
'*.so', 'lib/*.so*',
],
},
cmdclass={
'build_clib': CPPLibBuild,
'build_ext': Build,
'build_py': PythonPackageBuild,
'egg_info': EggInfoBuild,
'clean': Clean,
'install': InstallCmd
})
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Python
1
https://gitee.com/pengcheng111/pytorch.git
git@gitee.com:pengcheng111/pytorch.git
pengcheng111
pytorch
pytorch
master

搜索帮助