diff --git a/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile b/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile index bfc7f03462f31ba5e21a46208927101c8fc3cae0..f8cb4ef612ce4554fdf1e9c49d9b987397c1d631 100644 --- a/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile +++ b/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile @@ -33,29 +33,12 @@ RUN apt-get remove -y python3-blinker && apt-get autoremove -y # 安装cann 依赖 RUN pip3 install --upgrade setuptools && pip3 install numpy==1.26.4 decorator==5.1.1 sympy==1.12 cffi==1.16.0 pyyaml==6.0.1 pathlib2==2.3.7.post1 protobuf==5.26.0 scipy==1.12.0 requests==2.31.0 psutil==5.9.8 absl-py==2.1.0 attrs==23.2.0 -# 安装torch for x86 -# RUN pip3 install torch=="${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/cpu -# 安装torch-npu -# RUN pip3 install torch-npu=="${TORCH_VERSION}".post3 - -# 安装cann-toolkit和kernel -COPY ./package/Ascend-cann*_linux-${ARCH}.run /tmp/ RUN useradd -d /home/HwHiAiUser -u 1000 -m -s /bin/bash HwHiAiUser -RUN bash /tmp/Ascend-cann-toolkit*_linux-${ARCH}.run --install --install-for-all --quiet -RUN bash /tmp/Ascend-cann-kernels*_linux-${ARCH}.run --install --install-for-all --quiet -#安装 nnal -RUN bash -c "source /usr/local/Ascend/ascend-toolkit/set_env.sh && bash /tmp/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet" - -ENV ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest -ENV TOOLCHAIN_HOME=/usr/local/Ascend/ascend-toolkit/latest/toolkit -ENV ASCEND_TOOLKIT_HOME=/usr/local/Ascend/ascend-toolkit/latest -ENV ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp -ENV ASCEND_AICPU_PATH=/usr/local/Ascend/ascend-toolkit/latest + ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:$LD_LIBRARY_PATH RUN pip3 install build wheel - # 安装openblas ARG OPENBLAS_INSTALL_PATH=/usr/local/Ascend/OpenBLAS RUN . /tmp/urls.conf && cd /tmp && \ @@ -88,30 +71,13 @@ RUN . /tmp/urls.conf && wget $FAISS_URL && \ # 安装index COPY ./package/Ascend-mindxsdk-mxindex*_linux-${ARCH}.run /tmp/ RUN bash /tmp/Ascend-mindxsdk-mxindex*_linux-${ARCH}.run --quiet --install --install-path=/usr/local/Ascend --platform=${PLATFORM} -ENV MX_INDEX_MODELPATH=/home/ascend/modelpath -RUN cd /usr/local/Ascend/mxIndex/ops && ./custom_opp_${ARCH}.run && mkdir -p /home/ascend/modelpath -# 安装ascendfaiss,到这里了 + +# 安装ascendfaiss ARG FAISS_INSTALL_PATH=/usr/local/faiss/faiss1.10.0 ARG MXINDEX_INSTALL_PATH=/usr/local/Ascend/mxIndex -ARG PYTHON_HEADER=/usr/include/${PYTHON_VERSION}/ -ARG ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest -ARG DRIVER_INSTALL_PATH=/usr/local/Ascend COPY ./package/driver /usr/local/Ascend/driver -RUN . /tmp/urls.conf && wget $ASCENDFAISS_URL - - -RUN unzip master.zip && \ - cd mindsdk-referenceapps-master/IndexSDK/faiss-python && \ - swig -python -c++ -Doverride= -module swig_ascendfaiss -I${PYTHON_HEADER} -I${FAISS_INSTALL_PATH}/include -I${MXINDEX_INSTALL_PATH}/include -DSWIGWORDSIZE64 -o swig_ascendfaiss.cpp swig_ascendfaiss.swig && \ - g++ -std=c++11 -DFINTEGER=int -fopenmp -I/usr/local/include -I${ASCEND_INSTALL_PATH}/acllib/include -I${ASCEND_INSTALL_PATH}/runtime/include -I${DRIVER_INSTALL_PATH}/driver/kernel/inc/driver -I${DRIVER_INSTALL_PATH}/driver/kernel/libc_sec/include -fPIC -fstack-protector-all -Wall -Wreturn-type -D_FORTIFY_SOURCE=2 -g -O3 -Wall -Wextra -I${PYTHON_HEADER} -I/usr/local/lib/${PYTHON_VERSION}/dist-packages/numpy/core/include -I${FAISS_INSTALL_PATH}/include -I${MXINDEX_INSTALL_PATH}/include -c swig_ascendfaiss.cpp -o swig_ascendfaiss.o && \ - g++ -std=c++11 -shared -fopenmp -L${ASCEND_INSTALL_PATH}/lib64 -L${ASCEND_INSTALL_PATH}/acllib/lib64 -L${ASCEND_INSTALL_PATH}/runtime/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64/common -L${DRIVER_INSTALL_PATH}/driver/lib64/driver -L${FAISS_INSTALL_PATH}/lib -L${MXINDEX_INSTALL_PATH}/lib -Wl,-rpath-link=${ASCEND_INSTALL_PATH}/acllib/lib64:${ASCEND_INSTALL_PATH}/runtime/lib64:${DRIVER_INSTALL_PATH}/driver/lib64:${DRIVER_INSTALL_PATH}/driver/lib64/common:${DRIVER_INSTALL_PATH}/driver/lib64/driver -L/usr/local/lib -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -s -o _swig_ascendfaiss.so swig_ascendfaiss.o -L.. -lascendfaiss -lfaiss -lascend_hal -lacl_retr -lascendcl -lc_sec -lacl_op_compiler && \ - ${PYTHON_VERSION} -m build && \ - cd dist && pip3 install ascendfaiss*.whl - - -#ENV LD_LIBRARY_PATH=/usr/local/Ascend/mxIndex/lib:/usr/local/faiss/faiss1.10.0/lib:$LD_LIBRARY_PATH # 安装toch_npu @@ -122,20 +88,21 @@ RUN pip3 install torch-npu=="${TORCH_VERSION}".post8 COPY ./package/Ascend-mindxsdk-mxrag_*_linux-${ARCH}.run /tmp/ RUN bash /tmp/Ascend-mindxsdk-mxrag_*_linux-${ARCH}.run --install --install-path=/usr/local/Ascend --quiet --platform=${PLATFORM} RUN pip3 install -r /usr/local/Ascend/mxRag/requirements.txt - +RUN pip3 install torch-npu=="${TORCH_VERSION}".post12 pymilvus==2.5.12 # 安装mxrag第三方依赖 -RUN pip3 install ragas==0.1.9 rank_bm25==0.2.2 readability_lxml==0.8.1 html_text==0.6.2 gradio==3.50.2 +RUN pip3 install ragas==0.1.9 rank_bm25==0.2.2 readability_lxml==0.8.1 html_text==0.6.2 gradio==3.50.2 streamlit #清理临时目录 -RUN rm -rf ./* && rm -rf /usr/local/Ascend/driver # 添加环境变量 RUN sed -i '$a\export PYTHONPATH=/root/.local/lib/$PYTHON_VERSION/site-packages/mx_rag/libs/:$PYTHONPATH' /root/.bashrc && \ sed -i '$a\export LD_PRELOAD=$(ls /usr/local/lib/$PYTHON_VERSION/dist-packages/scikit_learn.libs/libgomp-*):$LD_PRELOAD' /root/.bashrc && \ - sed -i '$a\source /usr/local/Ascend/ascend-toolkit/set_env.sh' /root/.bashrc && \ - sed -i '$a\source /usr/local/Ascend/nnal/atb/set_env.sh' /root/.bashrc && \ sed -i '$a\source /usr/local/Ascend/mxRag/script/set_env.sh' /root/.bashrc && \ sed -i '$a\export LD_LIBRARY_PATH=/usr/local/Ascend/mxIndex/lib:/usr/local/faiss/faiss1.10.0/lib:$LD_LIBRARY_PATH' /root/.bashrc && \ sed -i '$a\export PATH=/usr/local/bin:$PATH' /root/.bashrc && \ + sed -i '$a\export ASCEND_HOME=/usr/local/Ascend' /root/.bashrc && \ + sed -i '$a\export ASCEND_VERSION=/usr/local/Ascend/ascend-toolkit/latest' /root/.bashrc && \ + sed -i '$a\export MX_INDEX_MODELPATH=/home/ascend/modelpath' /root/.bashrc && \ + sed -i '$a\export MX_INDEX_INSTALL_PATH=/usr/local/Ascend/mxIndex' /root/.bashrc && \ sed -i 's/$PYTHON_VERSION/'"$PYTHON_VERSION"'/g' /root/.bashrc && \ sed -i '$a\export LOGURU_FORMAT="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message!r}"' /root/.bashrc @@ -153,11 +120,11 @@ RUN bash /tmp/Ascend-mindxsdk-mxrag_*_linux-${ARCH}.run --install --install-path # 添加环境变量 for HwHiAiUser用户 RUN sed -i '$a\export PYTHONPATH=/home/HwHiAiUser/.local/lib/$PYTHON_VERSION/site-packages/mx_rag/libs/:$PYTHONPATH' /home/HwHiAiUser/.bashrc && \ sed -i '$a\export LD_PRELOAD=$(ls /usr/local/lib/$PYTHON_VERSION/dist-packages/scikit_learn.libs/libgomp-*):$LD_PRELOAD' /home/HwHiAiUser/.bashrc && \ - sed -i '$a\source /usr/local/Ascend/ascend-toolkit/set_env.sh' /home/HwHiAiUser/.bashrc && \ - sed -i '$a\source /usr/local/Ascend/nnal/atb/set_env.sh' /home/HwHiAiUser/.bashrc && \ sed -i '$a\source /home/HwHiAiUser/Ascend/mxRag/script/set_env.sh' /home/HwHiAiUser/.bashrc && \ sed -i '$a\export LD_LIBRARY_PATH=/home/HwHiAiUser/Ascend/mxIndex/lib:/usr/local/faiss/faiss1.10.0/lib:$LD_LIBRARY_PATH' /home/HwHiAiUser/.bashrc && \ sed -i '$a\export PATH=/usr/local/bin:$PATH' /home/HwHiAiUser/.bashrc && \ + sed -i '$a\export MX_INDEX_MODELPATH=/home/HwHiAiUser/Ascend/modelpath' /home/HwHiAiUser/.bashrc && \ + sed -i '$a\export MX_INDEX_INSTALL_PATH=/home/HwHiAiUser/Ascend/mxIndex' /home/HwHiAiUser/.bashrc&& \ sed -i 's/$PYTHON_VERSION/'"$PYTHON_VERSION"'/g' /home/HwHiAiUser/.bashrc && \ sed -i '$a\export LOGURU_FORMAT="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message!r}"' /home/HwHiAiUser/.bashrc @@ -165,9 +132,12 @@ ENV MX_INDEX_MULTITHREAD=1 ENV MX_INDEX_FINALIZE=0 USER root -RUN chmod +r /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/config.ini +RUN mkdir -p /opt/package/ +COPY ./package/Ascend-cann*_linux-${ARCH}.run /opt/package/ +RUN . /tmp/urls.conf && wget $ASCENDFAISS_URL -P /opt/package/ +COPY ./package/install.sh /opt/package/ + RUN rm -rf /tmp/* USER HwHiAiUser:HwHiAiUser WORKDIR /home/HwHiAiUser - diff --git a/RAGSDK/MainRepo/Dockerfile/ubuntu/install.sh b/RAGSDK/MainRepo/Dockerfile/ubuntu/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..fa43ea6bbcd4f0ad0cc0ce91d2993676e9e4dc20 --- /dev/null +++ b/RAGSDK/MainRepo/Dockerfile/ubuntu/install.sh @@ -0,0 +1,67 @@ +#!/bin/bash +ARCH=$(uname -m) + +if [[ -n $(id | grep uid=0) ]];then + usr_id=0 +else + usr_id=1 +fi + +# 安装cann-toolkit和kernel +bash /opt/package/Ascend-cann-toolkit*_linux-${ARCH}.run --install --install-for-all --quiet +bash /opt/package/Ascend-cann-kernels*_linux-${ARCH}.run --install --install-for-all --quiet +#安装 nnal +if [[ $usr_id -eq 0 ]]; then + bash -c "source /usr/local/Ascend/ascend-toolkit/set_env.sh && bash /opt/package/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet" + + commands=("source /usr/local/Ascend/ascend-toolkit/set_env.sh" "source /usr/local/Ascend/nnal/atb/set_env.sh" "export ASCEND_VERSION=/usr/local/Ascend/ascend-toolkit/latest" "export ASCEND_HOME=/usr/local/Ascend") + for cmd in "${commands[@]}"; + do + if ! grep -Fxq "$cmd" "/root/.bashrc"; then + # 如果不存在,将命令追加到文件中 + echo "$cmd" >> "/root/.bashrc" + echo "命令已添加到 /root/.bashrc" + fi + done; + + source /usr/local/Ascend/ascend-toolkit/set_env.sh + source /usr/local/Ascend/nnal/atb/set_env.sh + export ASCEND_VERSION=/usr/local/Ascend/ascend-toolkit/latest + export ASCEND_HOME=/usr/local/Ascend + bash /opt/package/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet + + cd /usr/local/Ascend/mxIndex/ops && ./custom_opp_${ARCH}.run && mkdir -p ${MX_INDEX_MODELPATH} +else + bash -c "source /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh && bash /opt/package/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet" + commands=("source /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh" "source /home/HwHiAiUser/Ascend/nnal/atb/set_env.sh" "export ASCEND_HOME=/home/HwHiAiUser/Ascend" "export ASCEND_VERSION=/home/HwHiAiUser/Ascend/ascend-toolkit/latest") + for cmd in "${commands[@]}"; + do + if ! grep -Fxq "$cmd" "/home/HwHiAiUser/.bashrc"; then + # 如果不存在,将命令追加到文件中 + echo "$cmd" >> "/home/HwHiAiUser/.bashrc" + echo "命令已添加到 /home/HwHiAiUser/.bashrc" + fi + done; + + source /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh + source /home/HwHiAiUser/Ascend/nnal/atb/set_env.sh + export ASCEND_HOME=/home/HwHiAiUser/Ascend + export ASCEND_VERSION=/home/HwHiAiUser/Ascend/ascend-toolkit/latest + + cd /home/HwHiAiUser/Ascend/mxIndex/ops && ./custom_opp_${ARCH}.run && mkdir -p ${MX_INDEX_MODELPATH} +fi + + +PYTHON_VERSION=python3.11 +PYTHON_HEADER=/usr/include/${PYTHON_VERSION}/ +FAISS_INSTALL_PATH=/usr/local/faiss/faiss1.10.0 +DRIVER_INSTALL_PATH=/usr/local/Ascend + + +rm -rf /tmp/mindsdk-referenceapps-master && unzip -d /tmp /opt/package/master.zip && \ + cd /tmp/mindsdk-referenceapps-master/IndexSDK/faiss-python && \ + swig -python -c++ -Doverride= -module swig_ascendfaiss -I${PYTHON_HEADER} -I${FAISS_INSTALL_PATH}/include -I${MX_INDEX_INSTALL_PATH}/include -DSWIGWORDSIZE64 -o swig_ascendfaiss.cpp swig_ascendfaiss.swig && \ + g++ -std=c++11 -DFINTEGER=int -fopenmp -I/usr/local/include -I${ASCEND_TOOLKIT_HOME}/acllib/include -I${ASCEND_TOOLKIT_HOME}/runtime/include -fPIC -fstack-protector-all -Wall -Wreturn-type -D_FORTIFY_SOURCE=2 -g -O3 -Wall -Wextra -I${PYTHON_HEADER} -I/usr/local/lib/${PYTHON_VERSION}/dist-packages/numpy/core/include -I${FAISS_INSTALL_PATH}/include -I${MX_INDEX_INSTALL_PATH}/include -c swig_ascendfaiss.cpp -o swig_ascendfaiss.o && \ + g++ -std=c++11 -shared -fopenmp -L${ASCEND_TOOLKIT_HOME}/lib64 -L${ASCEND_TOOLKIT_HOME}/acllib/lib64 -L${ASCEND_TOOLKIT_HOME}/runtime/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64/common -L${DRIVER_INSTALL_PATH}/driver/lib64/driver -L${FAISS_INSTALL_PATH}/lib -L${MX_INDEX_INSTALL_PATH}/lib -Wl,-rpath-link=${ASCEND_TOOLKIT_HOME}/acllib/lib64:${ASCEND_TOOLKIT_HOME}/runtime/lib64:${DRIVER_INSTALL_PATH}/driver/lib64:${DRIVER_INSTALL_PATH}/driver/lib64/common:${DRIVER_INSTALL_PATH}/driver/lib64/driver -L/usr/local/lib -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -s -o _swig_ascendfaiss.so swig_ascendfaiss.o -L.. -lascendfaiss -lfaiss -lascend_hal -lc_sec && \ + ${PYTHON_VERSION} -m build --no-isolation && \ + cd dist && pip3 install ascendfaiss*.whl && rm -rf /tmp/mindsdk-referenceapps-master/ \ No newline at end of file