From 0f201ed0c8d8f0d69535e72b47d818c3607bb1f7 Mon Sep 17 00:00:00 2001 From: lijiaming <1228575330@qq.com> Date: Wed, 16 Apr 2025 17:27:01 +0800 Subject: [PATCH 1/2] adjust int4 start command --- ...-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" | 4 ++-- .../workspace/roles/prepare/files/lib/start_ds.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git "a/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" "b/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" index dd60db3..2e74906 100644 --- "a/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" +++ "b/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" @@ -142,8 +142,8 @@ all: vars: # 容器镜像 # 如果本地docker上已加载镜像，改成docker image的image_name和image_tag - image_name: hub.oepkgs.net/oedeploy/openeuler/aarch64/deepseek_hyperinfer - image_tag: openeuler22.03-py3.11 + image_name: hub.oepkgs.net/oedeploy/openeuler/aarch64/mindspore + image_tag: 20250415 # 单机部署时，建议使用tag为20250326的容器镜像 # 将要拉起的推理容器的名称 container_name: openeuler_ds # 启动之后的docker name，不能和已有镜像重名 # 模型路径 diff --git a/script/mindspore-deepseek/workspace/roles/prepare/files/lib/start_ds.sh b/script/mindspore-deepseek/workspace/roles/prepare/files/lib/start_ds.sh index 49a7bd2..89209c4 100644 --- a/script/mindspore-deepseek/workspace/roles/prepare/files/lib/start_ds.sh +++ b/script/mindspore-deepseek/workspace/roles/prepare/files/lib/start_ds.sh @@ -39,7 +39,7 @@ rm -rf ds.log if [ $NODE_NUM -ne 1 ]; then nohup python3 -m vllm_mindspore.entrypoints vllm.entrypoints.openai.api_server --model "$MODEL_PATH" --port=$LLM_PORT --trust_remote_code --tensor_parallel_size=$PARALLEL --max-num-seqs=192 --max_model_len=32768 --max-num-batched-tokens=16384 --block-size=128 --gpu-memory-utilization=0.93 --num-scheduler-steps=8 --disable-async-output-proc --distributed-executor-backend=ray &> ds.log & else - nohup python3 -m vllm_mindspore.entrypoints vllm.entrypoints.openai.api_server --model "$MODEL_PATH" --port=$LLM_PORT --trust_remote_code --tensor_parallel_size=8 --max-num-seqs=192 --max_model_len=32768 --max-num-batched-tokens=16384 --block-size=128 --gpu-memory-utilization=0.93 --num-scheduler-steps=8 --disable-async-output-proc &> ds.log & + nohup python3 -m vllm_mindspore.entrypoints vllm.entrypoints.openai.api_server --model "$MODEL_PATH" --port=$LLM_PORT --trust_remote_code --tensor_parallel_size=8 --max-num-seqs=192 --max_model_len=32768 --max-num-batched-tokens=16384 --block-size=32 --gpu-memory-utilization=0.93 --num-scheduler-steps=8 --disable-async-output-proc &> ds.log & fi #检测推理服务是否拉起 -- Gitee From e64f718559f564bf7aed4f4323b3822bab163f1e Mon Sep 17 00:00:00 2001 From: lijiaming <1228575330@qq.com> Date: Wed, 16 Apr 2025 17:31:52 +0800 Subject: [PATCH 2/2] update readme --- README.md | 6 ++++-- ...3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a086c93..75b3f09 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,8 @@ size的动态变化，相比于静态shape的整图下沉，动态shape的计算 | OpenAI o1-mini | 68.9 | | DeepSeek R1 | 91.8 | | Deepseek R1 w8a8 | 89.52 | +| Deepseek R1 W4A16 | 88.78 | +| Deepseek V3 0324 W4A16 | 87.82 | @@ -141,8 +143,8 @@ size的动态变化，相比于静态shape的整图下沉，动态shape的计算 | 并发数 | 吞吐(Token/s) | | ------ | ------------- | -| 1 | 16.7 | -| 192 | 1188 | +| 1 | 22.4 | +| 192 | 1600 | diff --git "a/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" "b/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" index 2e74906..5515b29 100644 --- "a/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" +++ "b/doc/deepseek/DeepSeek-V3&R1\351\203\250\347\275\262\346\214\207\345\215\227.md" @@ -120,6 +120,7 @@ cd llm_solution/script/mindspore-deepseek ```shell # 调整mindspore-deepseek目录下config.yaml +# 注意：w8a8与int4不同权重部署方式，使用的镜像tag不一致，可按照下文修改 (base) [root@910b-3 mindspore-deepseek]# cat config.yaml all: children: @@ -143,7 +144,7 @@ all: # 容器镜像 # 如果本地docker上已加载镜像，改成docker image的image_name和image_tag image_name: hub.oepkgs.net/oedeploy/openeuler/aarch64/mindspore - image_tag: 20250415 # 单机部署时，建议使用tag为20250326的容器镜像 + image_tag: 20250415 # 单机部署时，请使用tag为20250326的容器镜像 # 将要拉起的推理容器的名称 container_name: openeuler_ds # 启动之后的docker name，不能和已有镜像重名 # 模型路径 -- Gitee