代码拉取完成,页面将自动刷新
同步操作将从 衣沾不足惜/gitee-ai-docs-test 强制同步,此操作会覆盖自 Fork 仓库以来所做的任何修改,且无法恢复!!!
确定后同步将在后台操作,完成时将刷新页面,请耐心等待。
import subprocess
import os
import time
import torch
import utils
model_name = "hf-models/Qwen2.5-7B-Instruct"
GITEE_ACCESS_TOKEN = os.environ.get("GITEE_ACCESS_TOKEN", "")
# print("开始下载 lora")
# git_clone_docs_lora_command = ["git", "clone",
# f"https://oauth2:{GITEE_ACCESS_TOKEN}@gitee.com/stringify/glm4-lora-gitee-docs-9b.git", "--depth=1", "--single-branch", "./glm4-lora-gitee-docs-9b"]
# subprocess.run(
# git_clone_docs_lora_command, text=True)
# print("lora 下载完成")
# subprocess.Popen(['pip','uninstall','-y','flash_attn'])
os.environ['HF_HOME'] = '/data'
api_server_command = [
"python",
"-m",
"vllm.entrypoints.openai.api_server",
"--model",
model_name,
"--dtype",
"bfloat16",
"--api-key",
"",
"--tensor-parallel-size",
str(torch.cuda.device_count() or 2),
"--trust-remote-code",
"--gpu-memory-utilization",
"0.71",
"--max-num-batched-tokens",
# "--enable-lora",
# "--lora-modules",
# "gitee-docs-lora=./glm4-lora-gitee-docs-9b", # vllm 0.3.3 不支持 glm4 qwen2 lora: ValueError: Model Qwen2ForCausalLM does not support LoRA, but LoRA is enabled. Support for this model may be added in the future. If this is important to you, please open an issue on github.
# ValueError: max_num_batched_tokens (55000) is smaller than max_model_len (131072). This effectively limits the maximum sequence length to max_num_batched_tokens and makes vLLM reject longer sequences. Please increase max_num_batched_tokens or decrease max_model_len.
"21000",
"--max-model-len",
"21000",
"--disable-log-requests",
"--disable-log-stats",
"--port",
"8000",
"--block-size",
"16", # 0.33 只支持16 默认 16
"--max-num-seqs",
"1024"
# "--enable-chunked-prefill", # 0.3.3 不支持启用分块预填充
# 多卡跑多 模型时, vllm GPU blocks: 0 https://github.com/vllm-project/vllm/issues/2248
# "--enforce-eager"
]
chainlit_ui_process = subprocess.Popen(
['python', '-m', 'chainlit', 'run', 'chainlit_ui_crawler.py', '--host', '0.0.0.0', '--port', '7860', "--ci", "--headless"])
def wait_for_service(url):
while True:
try:
server_ready = utils.is_port_open(url)
if server_ready:
return True
except:
time.sleep(5)
if (wait_for_service("http://127.0.0.1:7860")):
api_process = subprocess.Popen(
api_server_command, text=True)
print("UI 服务已启动,开始启动 API 服务...")
print("开始启动 api 服务")
# device = torch.cuda.current_device()
# props = torch.cuda.get_device_properties(device)
# print(f"Device Name: {props.name}")
# print(f"Total Memory: {props.total_memory / (1024 ** 3)} GB")
try:
api_process.wait()
chainlit_ui_process.wait()
# except KeyboardInterrupt:
# print("Shutting down servers.")
# chainlit_ui_process.terminate()
# api_process.terminate()
# api_process.wait()
# chainlit_ui_process.wait()
finally:
api_process.kill()
chainlit_ui_process.kill()
print("Servers shut down.")
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。