diff --git a/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile b/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile
index bfc7f03462f31ba5e21a46208927101c8fc3cae0..f8cb4ef612ce4554fdf1e9c49d9b987397c1d631 100644
--- a/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile
+++ b/RAGSDK/MainRepo/Dockerfile/ubuntu/Dockerfile
@@ -33,29 +33,12 @@ RUN apt-get remove -y python3-blinker && apt-get autoremove -y
# 安装cann 依赖
RUN pip3 install --upgrade setuptools && pip3 install numpy==1.26.4 decorator==5.1.1 sympy==1.12 cffi==1.16.0 pyyaml==6.0.1 pathlib2==2.3.7.post1 protobuf==5.26.0 scipy==1.12.0 requests==2.31.0 psutil==5.9.8 absl-py==2.1.0 attrs==23.2.0
-# 安装torch for x86
-# RUN pip3 install torch=="${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/cpu
-# 安装torch-npu
-# RUN pip3 install torch-npu=="${TORCH_VERSION}".post3
-
-# 安装cann-toolkit和kernel
-COPY ./package/Ascend-cann*_linux-${ARCH}.run /tmp/
RUN useradd -d /home/HwHiAiUser -u 1000 -m -s /bin/bash HwHiAiUser
-RUN bash /tmp/Ascend-cann-toolkit*_linux-${ARCH}.run --install --install-for-all --quiet
-RUN bash /tmp/Ascend-cann-kernels*_linux-${ARCH}.run --install --install-for-all --quiet
-#安装 nnal
-RUN bash -c "source /usr/local/Ascend/ascend-toolkit/set_env.sh && bash /tmp/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet"
-
-ENV ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest
-ENV TOOLCHAIN_HOME=/usr/local/Ascend/ascend-toolkit/latest/toolkit
-ENV ASCEND_TOOLKIT_HOME=/usr/local/Ascend/ascend-toolkit/latest
-ENV ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp
-ENV ASCEND_AICPU_PATH=/usr/local/Ascend/ascend-toolkit/latest
+
ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64/common:/usr/local/Ascend/driver/lib64/driver:$LD_LIBRARY_PATH
RUN pip3 install build wheel
-
# 安装openblas
ARG OPENBLAS_INSTALL_PATH=/usr/local/Ascend/OpenBLAS
RUN . /tmp/urls.conf && cd /tmp && \
@@ -88,30 +71,13 @@ RUN . /tmp/urls.conf && wget $FAISS_URL && \
# 安装index
COPY ./package/Ascend-mindxsdk-mxindex*_linux-${ARCH}.run /tmp/
RUN bash /tmp/Ascend-mindxsdk-mxindex*_linux-${ARCH}.run --quiet --install --install-path=/usr/local/Ascend --platform=${PLATFORM}
-ENV MX_INDEX_MODELPATH=/home/ascend/modelpath
-RUN cd /usr/local/Ascend/mxIndex/ops && ./custom_opp_${ARCH}.run && mkdir -p /home/ascend/modelpath
-# 安装ascendfaiss,到这里了
+
+# 安装ascendfaiss
ARG FAISS_INSTALL_PATH=/usr/local/faiss/faiss1.10.0
ARG MXINDEX_INSTALL_PATH=/usr/local/Ascend/mxIndex
-ARG PYTHON_HEADER=/usr/include/${PYTHON_VERSION}/
-ARG ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest
-ARG DRIVER_INSTALL_PATH=/usr/local/Ascend
COPY ./package/driver /usr/local/Ascend/driver
-RUN . /tmp/urls.conf && wget $ASCENDFAISS_URL
-
-
-RUN unzip master.zip && \
- cd mindsdk-referenceapps-master/IndexSDK/faiss-python && \
- swig -python -c++ -Doverride= -module swig_ascendfaiss -I${PYTHON_HEADER} -I${FAISS_INSTALL_PATH}/include -I${MXINDEX_INSTALL_PATH}/include -DSWIGWORDSIZE64 -o swig_ascendfaiss.cpp swig_ascendfaiss.swig && \
- g++ -std=c++11 -DFINTEGER=int -fopenmp -I/usr/local/include -I${ASCEND_INSTALL_PATH}/acllib/include -I${ASCEND_INSTALL_PATH}/runtime/include -I${DRIVER_INSTALL_PATH}/driver/kernel/inc/driver -I${DRIVER_INSTALL_PATH}/driver/kernel/libc_sec/include -fPIC -fstack-protector-all -Wall -Wreturn-type -D_FORTIFY_SOURCE=2 -g -O3 -Wall -Wextra -I${PYTHON_HEADER} -I/usr/local/lib/${PYTHON_VERSION}/dist-packages/numpy/core/include -I${FAISS_INSTALL_PATH}/include -I${MXINDEX_INSTALL_PATH}/include -c swig_ascendfaiss.cpp -o swig_ascendfaiss.o && \
- g++ -std=c++11 -shared -fopenmp -L${ASCEND_INSTALL_PATH}/lib64 -L${ASCEND_INSTALL_PATH}/acllib/lib64 -L${ASCEND_INSTALL_PATH}/runtime/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64/common -L${DRIVER_INSTALL_PATH}/driver/lib64/driver -L${FAISS_INSTALL_PATH}/lib -L${MXINDEX_INSTALL_PATH}/lib -Wl,-rpath-link=${ASCEND_INSTALL_PATH}/acllib/lib64:${ASCEND_INSTALL_PATH}/runtime/lib64:${DRIVER_INSTALL_PATH}/driver/lib64:${DRIVER_INSTALL_PATH}/driver/lib64/common:${DRIVER_INSTALL_PATH}/driver/lib64/driver -L/usr/local/lib -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -s -o _swig_ascendfaiss.so swig_ascendfaiss.o -L.. -lascendfaiss -lfaiss -lascend_hal -lacl_retr -lascendcl -lc_sec -lacl_op_compiler && \
- ${PYTHON_VERSION} -m build && \
- cd dist && pip3 install ascendfaiss*.whl
-
-
-#ENV LD_LIBRARY_PATH=/usr/local/Ascend/mxIndex/lib:/usr/local/faiss/faiss1.10.0/lib:$LD_LIBRARY_PATH
# 安装toch_npu
@@ -122,20 +88,21 @@ RUN pip3 install torch-npu=="${TORCH_VERSION}".post8
COPY ./package/Ascend-mindxsdk-mxrag_*_linux-${ARCH}.run /tmp/
RUN bash /tmp/Ascend-mindxsdk-mxrag_*_linux-${ARCH}.run --install --install-path=/usr/local/Ascend --quiet --platform=${PLATFORM}
RUN pip3 install -r /usr/local/Ascend/mxRag/requirements.txt
-
+RUN pip3 install torch-npu=="${TORCH_VERSION}".post12 pymilvus==2.5.12
# 安装mxrag第三方依赖
-RUN pip3 install ragas==0.1.9 rank_bm25==0.2.2 readability_lxml==0.8.1 html_text==0.6.2 gradio==3.50.2
+RUN pip3 install ragas==0.1.9 rank_bm25==0.2.2 readability_lxml==0.8.1 html_text==0.6.2 gradio==3.50.2 streamlit
#清理临时目录
-RUN rm -rf ./* && rm -rf /usr/local/Ascend/driver
# 添加环境变量
RUN sed -i '$a\export PYTHONPATH=/root/.local/lib/$PYTHON_VERSION/site-packages/mx_rag/libs/:$PYTHONPATH' /root/.bashrc && \
sed -i '$a\export LD_PRELOAD=$(ls /usr/local/lib/$PYTHON_VERSION/dist-packages/scikit_learn.libs/libgomp-*):$LD_PRELOAD' /root/.bashrc && \
- sed -i '$a\source /usr/local/Ascend/ascend-toolkit/set_env.sh' /root/.bashrc && \
- sed -i '$a\source /usr/local/Ascend/nnal/atb/set_env.sh' /root/.bashrc && \
sed -i '$a\source /usr/local/Ascend/mxRag/script/set_env.sh' /root/.bashrc && \
sed -i '$a\export LD_LIBRARY_PATH=/usr/local/Ascend/mxIndex/lib:/usr/local/faiss/faiss1.10.0/lib:$LD_LIBRARY_PATH' /root/.bashrc && \
sed -i '$a\export PATH=/usr/local/bin:$PATH' /root/.bashrc && \
+ sed -i '$a\export ASCEND_HOME=/usr/local/Ascend' /root/.bashrc && \
+ sed -i '$a\export ASCEND_VERSION=/usr/local/Ascend/ascend-toolkit/latest' /root/.bashrc && \
+ sed -i '$a\export MX_INDEX_MODELPATH=/home/ascend/modelpath' /root/.bashrc && \
+ sed -i '$a\export MX_INDEX_INSTALL_PATH=/usr/local/Ascend/mxIndex' /root/.bashrc && \
sed -i 's/$PYTHON_VERSION/'"$PYTHON_VERSION"'/g' /root/.bashrc && \
sed -i '$a\export LOGURU_FORMAT="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message!r}"' /root/.bashrc
@@ -153,11 +120,11 @@ RUN bash /tmp/Ascend-mindxsdk-mxrag_*_linux-${ARCH}.run --install --install-path
# 添加环境变量 for HwHiAiUser用户
RUN sed -i '$a\export PYTHONPATH=/home/HwHiAiUser/.local/lib/$PYTHON_VERSION/site-packages/mx_rag/libs/:$PYTHONPATH' /home/HwHiAiUser/.bashrc && \
sed -i '$a\export LD_PRELOAD=$(ls /usr/local/lib/$PYTHON_VERSION/dist-packages/scikit_learn.libs/libgomp-*):$LD_PRELOAD' /home/HwHiAiUser/.bashrc && \
- sed -i '$a\source /usr/local/Ascend/ascend-toolkit/set_env.sh' /home/HwHiAiUser/.bashrc && \
- sed -i '$a\source /usr/local/Ascend/nnal/atb/set_env.sh' /home/HwHiAiUser/.bashrc && \
sed -i '$a\source /home/HwHiAiUser/Ascend/mxRag/script/set_env.sh' /home/HwHiAiUser/.bashrc && \
sed -i '$a\export LD_LIBRARY_PATH=/home/HwHiAiUser/Ascend/mxIndex/lib:/usr/local/faiss/faiss1.10.0/lib:$LD_LIBRARY_PATH' /home/HwHiAiUser/.bashrc && \
sed -i '$a\export PATH=/usr/local/bin:$PATH' /home/HwHiAiUser/.bashrc && \
+ sed -i '$a\export MX_INDEX_MODELPATH=/home/HwHiAiUser/Ascend/modelpath' /home/HwHiAiUser/.bashrc && \
+ sed -i '$a\export MX_INDEX_INSTALL_PATH=/home/HwHiAiUser/Ascend/mxIndex' /home/HwHiAiUser/.bashrc&& \
sed -i 's/$PYTHON_VERSION/'"$PYTHON_VERSION"'/g' /home/HwHiAiUser/.bashrc && \
sed -i '$a\export LOGURU_FORMAT="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message!r}"' /home/HwHiAiUser/.bashrc
@@ -165,9 +132,12 @@ ENV MX_INDEX_MULTITHREAD=1
ENV MX_INDEX_FINALIZE=0
USER root
-RUN chmod +r /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/config.ini
+RUN mkdir -p /opt/package/
+COPY ./package/Ascend-cann*_linux-${ARCH}.run /opt/package/
+RUN . /tmp/urls.conf && wget $ASCENDFAISS_URL -P /opt/package/
+COPY ./package/install.sh /opt/package/
+
RUN rm -rf /tmp/*
USER HwHiAiUser:HwHiAiUser
WORKDIR /home/HwHiAiUser
-
diff --git a/RAGSDK/MainRepo/Dockerfile/ubuntu/install.sh b/RAGSDK/MainRepo/Dockerfile/ubuntu/install.sh
new file mode 100644
index 0000000000000000000000000000000000000000..fa43ea6bbcd4f0ad0cc0ce91d2993676e9e4dc20
--- /dev/null
+++ b/RAGSDK/MainRepo/Dockerfile/ubuntu/install.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+ARCH=$(uname -m)
+
+if [[ -n $(id | grep uid=0) ]];then
+ usr_id=0
+else
+ usr_id=1
+fi
+
+# 安装cann-toolkit和kernel
+bash /opt/package/Ascend-cann-toolkit*_linux-${ARCH}.run --install --install-for-all --quiet
+bash /opt/package/Ascend-cann-kernels*_linux-${ARCH}.run --install --install-for-all --quiet
+#安装 nnal
+if [[ $usr_id -eq 0 ]]; then
+ bash -c "source /usr/local/Ascend/ascend-toolkit/set_env.sh && bash /opt/package/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet"
+
+ commands=("source /usr/local/Ascend/ascend-toolkit/set_env.sh" "source /usr/local/Ascend/nnal/atb/set_env.sh" "export ASCEND_VERSION=/usr/local/Ascend/ascend-toolkit/latest" "export ASCEND_HOME=/usr/local/Ascend")
+ for cmd in "${commands[@]}";
+ do
+ if ! grep -Fxq "$cmd" "/root/.bashrc"; then
+ # 如果不存在,将命令追加到文件中
+ echo "$cmd" >> "/root/.bashrc"
+ echo "命令已添加到 /root/.bashrc"
+ fi
+ done;
+
+ source /usr/local/Ascend/ascend-toolkit/set_env.sh
+ source /usr/local/Ascend/nnal/atb/set_env.sh
+ export ASCEND_VERSION=/usr/local/Ascend/ascend-toolkit/latest
+ export ASCEND_HOME=/usr/local/Ascend
+ bash /opt/package/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet
+
+ cd /usr/local/Ascend/mxIndex/ops && ./custom_opp_${ARCH}.run && mkdir -p ${MX_INDEX_MODELPATH}
+else
+ bash -c "source /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh && bash /opt/package/Ascend-cann-nnal*_linux-${ARCH}.run --install --quiet"
+ commands=("source /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh" "source /home/HwHiAiUser/Ascend/nnal/atb/set_env.sh" "export ASCEND_HOME=/home/HwHiAiUser/Ascend" "export ASCEND_VERSION=/home/HwHiAiUser/Ascend/ascend-toolkit/latest")
+ for cmd in "${commands[@]}";
+ do
+ if ! grep -Fxq "$cmd" "/home/HwHiAiUser/.bashrc"; then
+ # 如果不存在,将命令追加到文件中
+ echo "$cmd" >> "/home/HwHiAiUser/.bashrc"
+ echo "命令已添加到 /home/HwHiAiUser/.bashrc"
+ fi
+ done;
+
+ source /home/HwHiAiUser/Ascend/ascend-toolkit/set_env.sh
+ source /home/HwHiAiUser/Ascend/nnal/atb/set_env.sh
+ export ASCEND_HOME=/home/HwHiAiUser/Ascend
+ export ASCEND_VERSION=/home/HwHiAiUser/Ascend/ascend-toolkit/latest
+
+ cd /home/HwHiAiUser/Ascend/mxIndex/ops && ./custom_opp_${ARCH}.run && mkdir -p ${MX_INDEX_MODELPATH}
+fi
+
+
+PYTHON_VERSION=python3.11
+PYTHON_HEADER=/usr/include/${PYTHON_VERSION}/
+FAISS_INSTALL_PATH=/usr/local/faiss/faiss1.10.0
+DRIVER_INSTALL_PATH=/usr/local/Ascend
+
+
+rm -rf /tmp/mindsdk-referenceapps-master && unzip -d /tmp /opt/package/master.zip && \
+ cd /tmp/mindsdk-referenceapps-master/IndexSDK/faiss-python && \
+ swig -python -c++ -Doverride= -module swig_ascendfaiss -I${PYTHON_HEADER} -I${FAISS_INSTALL_PATH}/include -I${MX_INDEX_INSTALL_PATH}/include -DSWIGWORDSIZE64 -o swig_ascendfaiss.cpp swig_ascendfaiss.swig && \
+ g++ -std=c++11 -DFINTEGER=int -fopenmp -I/usr/local/include -I${ASCEND_TOOLKIT_HOME}/acllib/include -I${ASCEND_TOOLKIT_HOME}/runtime/include -fPIC -fstack-protector-all -Wall -Wreturn-type -D_FORTIFY_SOURCE=2 -g -O3 -Wall -Wextra -I${PYTHON_HEADER} -I/usr/local/lib/${PYTHON_VERSION}/dist-packages/numpy/core/include -I${FAISS_INSTALL_PATH}/include -I${MX_INDEX_INSTALL_PATH}/include -c swig_ascendfaiss.cpp -o swig_ascendfaiss.o && \
+ g++ -std=c++11 -shared -fopenmp -L${ASCEND_TOOLKIT_HOME}/lib64 -L${ASCEND_TOOLKIT_HOME}/acllib/lib64 -L${ASCEND_TOOLKIT_HOME}/runtime/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64 -L${DRIVER_INSTALL_PATH}/driver/lib64/common -L${DRIVER_INSTALL_PATH}/driver/lib64/driver -L${FAISS_INSTALL_PATH}/lib -L${MX_INDEX_INSTALL_PATH}/lib -Wl,-rpath-link=${ASCEND_TOOLKIT_HOME}/acllib/lib64:${ASCEND_TOOLKIT_HOME}/runtime/lib64:${DRIVER_INSTALL_PATH}/driver/lib64:${DRIVER_INSTALL_PATH}/driver/lib64/common:${DRIVER_INSTALL_PATH}/driver/lib64/driver -L/usr/local/lib -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack -s -o _swig_ascendfaiss.so swig_ascendfaiss.o -L.. -lascendfaiss -lfaiss -lascend_hal -lc_sec && \
+ ${PYTHON_VERSION} -m build --no-isolation && \
+ cd dist && pip3 install ascendfaiss*.whl && rm -rf /tmp/mindsdk-referenceapps-master/
\ No newline at end of file