From a0c7f8a2f5f4e85535c3345a162fee0a14a8ccec Mon Sep 17 00:00:00 2001 From: zhoufan2956 Date: Thu, 17 Oct 2024 11:19:50 +0800 Subject: [PATCH 1/3] DiT IP-Adapter open-sora stable_diffusion --- .../built-in/foundation/DiT/README.md | 27 +++++++++-- .../built-in/foundation/DiT/export_model.py | 21 +++++---- .../built-in/foundation/IP-Adapter/README.md | 25 ++++++++-- .../foundation/IP-Adapter/export_ts.py | 20 ++++---- .../built-in/foundation/open-sora/README.md | 20 ++++++++ .../foundation/open-sora/export_model.py | 11 +++-- .../foundation/stable_diffusion/README.md | 47 +++++++++++++++---- .../stable_diffusion_pipeline.py | 18 +++---- .../stable_diffusion_pipeline_parallel.py | 20 ++++---- 9 files changed, 155 insertions(+), 54 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/foundation/DiT/README.md b/MindIE/MindIE-Torch/built-in/foundation/DiT/README.md index 94fa3b3021..7b0c3377a1 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/DiT/README.md +++ b/MindIE/MindIE-Torch/built-in/foundation/DiT/README.md @@ -98,6 +98,24 @@ latent_size = image_size // 8 2. 模型转换，该步骤会生成编译之后的pt模型 + 执行命令查看芯片名称（$\{chip\_name\}）。 + + ``` + npu-smi info + #该设备芯片chip_name=310P3 (自行替换) + 回显如下： + +-------------------+-----------------+------------------------------------------------------+ + | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | + | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | + +===================+=================+======================================================+ + | 0 310P3 | OK | 15.8 42 0 / 0 | + | 0 0 | 0000:82:00.0 | 0 1074 / 21534 | + +===================+=================+======================================================+ + | 1 310P3 | OK | 15.4 43 0 / 0 | + | 0 1 | 0000:89:00.0 | 0 1070 / 21534 | + +===================+=================+======================================================+ + ``` + ```bash # Duo卡 python3 export_model.py \ @@ -105,7 +123,8 @@ latent_size = image_size // 8 --vae_model ./sd-vae-ft-mse \ --image_size 512 \ --device 0 \ - --soc Duo \ + --soc Ascend${chip_name} \ + --device_type Duo --output_dir ./models \ --parallel @@ -115,7 +134,8 @@ latent_size = image_size // 8 --vae_model ./sd-vae-ft-mse \ --image_size 512 \ --device 0 \ - --soc A2 \ + --soc Ascend${chip_name} \ + --device_type A2 --output_dir ./models ``` @@ -125,7 +145,8 @@ latent_size = image_size // 8 - --vae_model：vae的权重路径 - --image_size：分辨率，支持256和512。默认为512 - --device：使用哪张卡 - - --soc：soc_version，只支持Duo和A2 + - --soc：处理器型号。 + - --device_type: 设备形态，当前支持A2、Duo两种形态。 - --output_dir：pt模型输出目录 - --parallel：【可选】模型使用并行进行推理 diff --git a/MindIE/MindIE-Torch/built-in/foundation/DiT/export_model.py b/MindIE/MindIE-Torch/built-in/foundation/DiT/export_model.py index 2b5c11b056..f550f52282 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/DiT/export_model.py +++ b/MindIE/MindIE-Torch/built-in/foundation/DiT/export_model.py @@ -65,10 +65,13 @@ def parse_arguments() -> Namespace: parser.add_argument("--parallel", action="store_true", help="Use parallel during inference") parser.add_argument( "--soc", - type=str, - default="Duo", - choices=["Duo", "A2"], - help="soc_version" + help="soc_version." + ) + parser.add_argument( + "--device_type", + choices=["A2", "Duo"], + default="A2", + help="device_type." ) return parser.parse_args() @@ -182,13 +185,11 @@ def main(): device_id = args.device mindietorch.set_device(device_id) - if args.soc == "Duo": - soc_version = "Ascend310P3" - elif args.soc == "A2": - soc_version = "Ascend910B4" - else: - print("Unsupport soc_version") + if args.device_type != "Duo" and args.device_type != "A2": + print("unsupport device_type, please check!") return + else: + soc_version = args.soc export_dit(args, soc_version) export_vae(args, soc_version) mindietorch.finalize() diff --git a/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/README.md b/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/README.md index cd5293d331..a7f3f3e5cc 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/README.md +++ b/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/README.md @@ -115,7 +115,24 @@ # ip_ckpt ip_ckpt="IP-Adapter-weights/models/ip-adapter_sd15.bin" ``` - + 执行命令查看芯片名称（$\{chip\_name\}）。 + + ``` + npu-smi info + #该设备芯片chip_name=310P3 (自行替换) + 回显如下： + +-------------------+-----------------+------------------------------------------------------+ + | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | + | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | + +===================+=================+======================================================+ + | 0 310P3 | OK | 15.8 42 0 / 0 | + | 0 0 | 0000:82:00.0 | 0 1074 / 21534 | + +===================+=================+======================================================+ + | 1 310P3 | OK | 15.4 43 0 / 0 | + | 0 1 | 0000:89:00.0 | 0 1070 / 21534 | + +===================+=================+======================================================+ + ``` + 执行命令： ```bash @@ -127,7 +144,8 @@ --batch_size 1 \ --output_dir ./models \ --device 0 \ - --soc Duo + --soc Ascend${chip_name} \ + --device_type Duo ``` 参数说明： @@ -137,7 +155,8 @@ - --output_dir: 导出的模型输出目录 - batch_size：目前只支持batch为1 - --device：使用的NPU芯片，默认是0 - - soc：soc_version。Duo为310P3，A2为910B4。默认为Duo + - --soc：处理器型号。 + - --device_type: 设备形态，当前支持A2、Duo两种形态。 diff --git a/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py b/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py index d2222322c1..5292e03011 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py +++ b/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py @@ -61,9 +61,13 @@ def parse_arguments() -> Namespace: ) parser.add_argument( "--soc", - choices=["Duo", "A2"], - default="Duo", - help="soc version.", + help="soc_version." + ) + parser.add_argument( + "--device_type", + choices=["A2", "Duo"], + default="A2", + help="device_type." ) parser.add_argument( "--device", @@ -71,7 +75,6 @@ def parse_arguments() -> Namespace: default=0, help="NPU device", ) - return parser.parse_args() class ImageEncoderExport(torch.nn.Module): @@ -268,8 +271,9 @@ def main(args): if __name__ == "__main__": args = parse_arguments() - if args.soc == "Duo": - soc_version = "Ascend310P3" - elif args.soc == "A2": - soc_version = "Ascend910B4" + if args.device_type != "Duo" and args.device_type != "A2": + print("unsupport device_type, please check!") + return + else: + soc_version = args.soc main(args) \ No newline at end of file diff --git a/MindIE/MindIE-Torch/built-in/foundation/open-sora/README.md b/MindIE/MindIE-Torch/built-in/foundation/open-sora/README.md index 2ac2fc0550..ed74ff07d6 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/open-sora/README.md +++ b/MindIE/MindIE-Torch/built-in/foundation/open-sora/README.md @@ -67,6 +67,24 @@ Open Sora采用动态掩码策略等技术细节复现Sora，并已实现可变 2. 模型转换，该步骤会生成编译之后的pt模型 + 执行命令查看芯片名称（$\{chip\_name\}）。 + + ``` + npu-smi info + #该设备芯片chip_name=310P3 (自行替换) + 回显如下： + +-------------------+-----------------+------------------------------------------------------+ + | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | + | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | + +===================+=================+======================================================+ + | 0 310P3 | OK | 15.8 42 0 / 0 | + | 0 0 | 0000:82:00.0 | 0 1074 / 21534 | + +===================+=================+======================================================+ + | 1 310P3 | OK | 15.4 43 0 / 0 | + | 0 1 | 0000:89:00.0 | 0 1070 / 21534 | + +===================+=================+======================================================+ + ``` + ```bash python3 export_model.py \ --output_dir ./models \ @@ -74,6 +92,7 @@ Open Sora采用动态掩码策略等技术细节复现Sora，并已实现可变 --dit_model_path ./OpenSora-v1-HQ-16x512x512.pth \ --vae_model_path ./sd-vae-ft-ema \ --resolution 16x512x512 \ + --soc Ascend${chip_name} \ --device_id 0 ``` @@ -83,6 +102,7 @@ Open Sora采用动态掩码策略等技术细节复现Sora，并已实现可变 - --dit_model_path：dit的权重路径 - --vae_model_path：vae的权重路径 - --resolution：分辨率。支持256和512 + - --soc：处理器型号。 - --device_id：NPU芯片 - --output_dir：pt模型输出目录 diff --git a/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py b/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py index a872b0bab2..a79731943e 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py +++ b/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py @@ -46,6 +46,10 @@ def parse_arguments(): default='16x512x512', choices=['16x256x256', '16x512x512'] ) + parser.add_argument( + "--soc", + help="soc_version." + ) parser.add_argument( "--device_id", type=int, @@ -93,7 +97,7 @@ def export_textencoder(args, save_dir, batch_size): require_full_compilation=True, truncate_long_and_double=True, precision_policy=_enums.PrecisionPolicy.FP32, - soc_version="Ascend910B4", + soc_version=soc_version, optimization_level=0 ) torch.jit.save(compiled_model, compiled_path) @@ -162,7 +166,7 @@ def export_dit(args, save_dir, batch_size): require_full_compilation=True, truncate_long_and_double=True, precision_policy=_enums.PrecisionPolicy.FP16, - soc_version="Ascend910B4", + soc_version=soc_version, optimization_level=0 ) torch.jit.save(compiled_model, compiled_path) @@ -208,7 +212,7 @@ def export_vae(args, save_dir, batch_size): require_full_compilation=True, truncate_long_and_double=True, precision_policy=_enums.PrecisionPolicy.FP16, - soc_version="Ascend910B4", + soc_version=soc_version, optimization_level=0 ) torch.jit.save(compiled_model, compiled_path) @@ -217,6 +221,7 @@ def main(): args = parse_arguments() device_id = args.device_id save_dir = args.output_dir + soc_version = args.soc mindietorch.set_device(device_id) batch_size = 1 diff --git a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/README.md b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/README.md index 8845b059fe..de6ab14136 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/README.md +++ b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/README.md @@ -221,7 +221,25 @@ ``` 当前查到NUMA node是0，对应0-23，推荐绑定其中单核以获得更好的性能。 - 3. 执行推理脚本。 + 3. 执行命令查看芯片名称（$\{chip\_name\}）。 + + ``` + npu-smi info + #该设备芯片chip_name=310P3 (自行替换) + 回显如下： + +-------------------+-----------------+------------------------------------------------------+ + | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | + | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | + +===================+=================+======================================================+ + | 0 310P3 | OK | 15.8 42 0 / 0 | + | 0 0 | 0000:82:00.0 | 0 1074 / 21534 | + +===================+=================+======================================================+ + | 1 310P3 | OK | 15.4 43 0 / 0 | + | 0 1 | 0000:89:00.0 | 0 1070 / 21534 | + +===================+=================+======================================================+ + ``` + + 4. 执行推理脚本。 ```bash # 1.若不使用并行推理： # 1.1不使用lora权重 @@ -232,7 +250,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc A2 \ + --soc Ascend${chip_name} \ + --device_type A2 --output_dir ./models \ --use_cache # 1.2使用带lora权重的新权重 @@ -243,7 +262,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc A2 \ + --soc Ascend${chip_name} \ + --device_type A2 --output_dir ./models_lora \ --use_cache @@ -256,7 +276,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc Duo \ + --soc Ascend${chip_name} \ + --device_type Duo --output_dir ./models \ --use_cache # 2.2使用带lora权重的新权重 @@ -267,7 +288,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc Duo \ + --soc Ascend${chip_name} \ + --device_type Duo --output_dir ./models_lora \ --use_cache ``` @@ -279,7 +301,8 @@ - --steps：生成图片迭代次数。 - --device：推理设备ID；可用逗号分割传入两个设备ID，此时会使用并行方式进行推理。 - --scheduler: 【可选】推荐使用DDIM采样器。 - - --soc: 硬件配置，根据硬件配置选择Duo或者A2。A2特指910B4。 + - --soc：处理器型号。 + - --device_type: 设备形态，当前支持A2、Duo两种形态。 - --output_dir: 编译好的模型路径。 - --use_cache: 【可选】推荐使用UnetCache策略。 - --use_cache_faster: 【可选】模型使用deepcache+faster融合方案。 @@ -326,7 +349,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc A2 \ + --soc Ascend${chip_name} \ + --device_type A2 --output_dir ./models \ --use_cache # 1.2使用带lora权重的新权重 @@ -339,7 +363,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc A2 \ + --soc Ascend${chip_name} \ + --device_type A2 --output_dir ./models_lora \ --use_cache @@ -354,7 +379,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc Duo \ + --soc Ascend${chip_name} \ + --device_type Duo --output_dir ./models \ --use_cache # 2.2使用带lora权重的新权重 @@ -367,7 +393,8 @@ --save_dir ./results \ --steps 50 \ --scheduler DDIM \ - --soc Duo \ + --soc Ascend${chip_name} \ + --device_type Duo --output_dir ./models_lora \ --use_cache ``` diff --git a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline.py b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline.py index 16d986539b..a5c2ff8f3e 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline.py +++ b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline.py @@ -135,13 +135,11 @@ class AIEStableDiffusionPipeline(StableDiffusionPipeline): size = self.args.batch_size batch_size = self.args.batch_size * 2 - if self.args.soc == "Duo": - soc_version = "Ascend310P3" - elif self.args.soc == "A2": - soc_version = "Ascend910B4" - else: - print("unsupport soc_version, please check!") + if self.args.device_type != "Duo" and self.args.device_type != "A2": + print("unsupport device_type, please check!") return + else: + soc_version = self.args.soc clip_compiled_path = os.path.join(self.args.output_dir, f"clip/clip_bs{size}_aie_compile.ts") if os.path.exists(clip_compiled_path): @@ -982,9 +980,13 @@ def parse_arguments(): ) parser.add_argument( "--soc", - choices=["Duo", "A2"], + help="soc_version." + ) + parser.add_argument( + "--device_type", + choices=["A2", "Duo"], default="A2", - help="soc_version.", + help="device_type." ) parser.add_argument( "-o", diff --git a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py index 5e64cac270..bcad9006ce 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py +++ b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py @@ -151,13 +151,11 @@ class AIEStableDiffusionPipeline(StableDiffusionPipeline): batch_size = self.args.batch_size - if self.args.soc == "Duo": - soc_version = "Ascend310P3" - elif self.args.soc == "A2": - soc_version = "Ascend910B4" - else: - print("unsupport soc_version, please check!") + if self.args.device_type != "Duo" and self.args.device_type != "A2": + print("unsupport device_type, please check!") return + else: + soc_version = self.args.soc clip_compiled_path = os.path.join(self.args.output_dir, f"clip/clip_bs{batch_size}_aie_compile.ts") if os.path.exists(clip_compiled_path): @@ -1146,9 +1144,13 @@ def parse_arguments(): ) parser.add_argument( "--soc", - choices=["Duo", "A2"], - default="Duo", - help="soc_version.", + help="soc_version." + ) + parser.add_argument( + "--device_type", + choices=["A2", "Duo"], + default="A2", + help="device_type." ) parser.add_argument( "--use_cache", -- Gitee From 4d4c20aadb582ee5ef542f89a47878f853f8e8f1 Mon Sep 17 00:00:00 2001 From: zhoufan2956 Date: Thu, 17 Oct 2024 11:35:16 +0800 Subject: [PATCH 2/3] modify args --- .../MindIE-Torch/built-in/foundation/open-sora/export_model.py | 2 +- .../stable_diffusion/stable_diffusion_pipeline_parallel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py b/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py index a79731943e..45ef1762ca 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py +++ b/MindIE/MindIE-Torch/built-in/foundation/open-sora/export_model.py @@ -48,7 +48,7 @@ def parse_arguments(): ) parser.add_argument( "--soc", - help="soc_version." + help="soc_version" ) parser.add_argument( "--device_id", diff --git a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py index bcad9006ce..39eb553cea 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py +++ b/MindIE/MindIE-Torch/built-in/foundation/stable_diffusion/stable_diffusion_pipeline_parallel.py @@ -1149,7 +1149,7 @@ def parse_arguments(): parser.add_argument( "--device_type", choices=["A2", "Duo"], - default="A2", + default="Duo", help="device_type." ) parser.add_argument( -- Gitee From 1294b635f53d1b5e3f59a6678a3b7829005520af Mon Sep 17 00:00:00 2001 From: zhoufan2956 Date: Thu, 17 Oct 2024 14:38:19 +0800 Subject: [PATCH 3/3] modify code check --- MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py b/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py index 5292e03011..058241d16f 100644 --- a/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py +++ b/MindIE/MindIE-Torch/built-in/foundation/IP-Adapter/export_ts.py @@ -273,7 +273,6 @@ if __name__ == "__main__": args = parse_arguments() if args.device_type != "Duo" and args.device_type != "A2": print("unsupport device_type, please check!") - return else: soc_version = args.soc main(args) \ No newline at end of file -- Gitee