From 1b4d27133f8586ed190cd67ac129f1eb41a4c51a Mon Sep 17 00:00:00 2001 From: root Date: Wed, 7 Jul 2021 17:07:10 +0800 Subject: [PATCH 1/4] add openEuler apache dockerfiles --- .../Apache/Dockerfile-apache-hbase-2.2.4 | 53 +++++++++++++ .../Apache/Dockerfile-apache-hive-3.1.1 | 56 +++++++++++++ .../Apache/Dockerfile-hadoop-3.2.0 | 78 +++++++++++++++++++ .../Apache/Dockerfile-spark-2.4.6 | 56 +++++++++++++ .../Apache/Dockerfile-zookeeper-3.6.2 | 38 +++++++++ 5 files changed, 281 insertions(+) create mode 100644 BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hbase-2.2.4 create mode 100644 BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hive-3.1.1 create mode 100644 BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-hadoop-3.2.0 create mode 100644 BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-2.4.6 create mode 100644 BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-zookeeper-3.6.2 diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hbase-2.2.4 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hbase-2.2.4 new file mode 100644 index 0000000..d0e4547 --- /dev/null +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hbase-2.2.4 @@ -0,0 +1,53 @@ +FROM arm64v8/openeuler:20.03-lts-sp1 + +# install base libs +WORKDIR /opt/tools/installed +RUN yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ + && mv /usr/bin/gcc /usr/bin/gcc-impl \ + && echo -e '#! /bin/sh\n/usr/bin/gcc-impl -fsigned-char "$@"' >> /usr/bin/gcc \ + && mv /usr/bin/g++ /usr/bin/g++-impl \ + && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ + && chmod +x /usr/bin/gcc /usr/bin/g++ \ + && yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen libtirpc-devel patch vim unzip diffutils.aarch64 cyrus-sasl* libtirpc-devel libXt-devel.aarch64 readline-devel.aarch64 autoconf cppunit-devel hostname + +# install maven and jdk +WORKDIR /opt/tools/installed + # download openjdk from github +RUN wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && tar -zxf OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/apache-maven-3.5.4-bin.tar.gz \ + #https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz + && tar -zxf apache-maven-3.5.4-bin.tar.gz -C /opt/tools/installed/ \ + && sed -i "146 a \n huaweimaven\n huawei maven\n https://mirrors.huaweicloud.com/repository/maven/\n central\n" /opt/tools/installed/apache-maven-3.5.4/conf/settings.xml + # set ENV +ENV JAVA_HOME=/opt/tools/installed/jdk8u282-b08 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.5.4 +ENV PATH=$MAVEN_HOME/bin:$PATH + +# install protobuf 2.5.0 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/ENV/protobuf-2.5.0.tar.gz \ + # wget --no-check-certificate https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \ + && tar -xf protobuf-2.5.0.tar.gz \ + # && wget https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/bigdata/Patch/protoc.zip \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protoc.zip \ + && unzip protoc.zip \ + && cp ./protoc/protoc.patch ./protobuf-2.5.0/src/google/protobuf/stubs/ \ + && cd protobuf-2.5.0/src/google/protobuf/stubs/ \ + && patch -p1 < protoc.patch \ + && cd /opt/tools/installed/protobuf-2.5.0 \ + && ./autogen.sh && ./configure CFLAGS='-fsigned-char' && make -j8 && make install \ + && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc +WORKDIR /opt/tools/installed + +# compile hbase +RUN wget http://archive-server:50050/cache-data/BigData/Apache/hbase/hbase-2.2.4-src.tar.gz \ + # wget https://archive.apache.org/dist/hbase/2.2.4/hbase-2.2.4-src.tar.gz \ + && tar -zxf hbase-2.2.4-src.tar.gz \ + && cd hbase-2.2.4 \ + && sed -i '3856 a \n \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ + && sed -i '3862 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releases\n \n ' ./pom.xml \ + && mvn clean package -DskipTests assembly:single \ + && mkdir -p /root/output \ + && cp -r ./hbase-assembly/target/ /root/output diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hive-3.1.1 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hive-3.1.1 new file mode 100644 index 0000000..54274c5 --- /dev/null +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-apache-hive-3.1.1 @@ -0,0 +1,56 @@ +FROM arm64v8/openeuler:20.03-lts-sp1 + +# install base libs +WORKDIR /opt/tools/installed +RUN yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ + && mv /usr/bin/gcc /usr/bin/gcc-impl \ + && echo -e '#! /bin/sh\n/usr/bin/gcc-impl -fsigned-char "$@"' >> /usr/bin/gcc \ + && mv /usr/bin/g++ /usr/bin/g++-impl \ + && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ + && chmod +x /usr/bin/gcc /usr/bin/g++ \ + && yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen libtirpc-devel patch vim unzip diffutils.aarch64 cyrus-sasl* libtirpc-devel libXt-devel.aarch64 readline-devel.aarch64 autoconf cppunit-devel hostname + +# install maven and jdk +WORKDIR /opt/tools/installed + # download openjdk from github +RUN wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && tar -zxf OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/apache-maven-3.5.4-bin.tar.gz \ + #https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz + && tar -zxf apache-maven-3.5.4-bin.tar.gz -C /opt/tools/installed/ \ + && sed -i "146 a \n huaweimaven\n huawei maven\n https://mirrors.huaweicloud.com/repository/maven/\n central\n" /opt/tools/installed/apache-maven-3.5.4/conf/settings.xml + # set ENV +ENV JAVA_HOME=/opt/tools/installed/jdk8u282-b08 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.5.4 +ENV PATH=$MAVEN_HOME/bin:$PATH + +# install protobuf 2.5.0 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/ENV/protobuf-2.5.0.tar.gz \ + # wget --no-check-certificate https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \ + && tar -xf protobuf-2.5.0.tar.gz \ + # && wget https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/bigdata/Patch/protoc.zip \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protoc.zip \ + && unzip protoc.zip \ + && cp ./protoc/protoc.patch ./protobuf-2.5.0/src/google/protobuf/stubs/ \ + && cd protobuf-2.5.0/src/google/protobuf/stubs/ \ + && patch -p1 < protoc.patch \ + && cd /opt/tools/installed/protobuf-2.5.0 \ + && ./autogen.sh && ./configure CFLAGS='-fsigned-char' && make -j8 && make install \ + && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc + +# compile hive +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/hive/release-3.1.1.zip \ + # wget --no-check-certificate https://github.com/apache/hive/archive/refs/tags/rel/release-3.1.1.zip \ + && unzip release-3.1.1.zip \ + && cd /opt/tools/installed/hive-rel-release-3.1.1 \ + && sed -i '218 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ + && sed -i '223 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releases\n ' ./pom.xml \ + && sed -i '295 a \n \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./upgrade-acid/pom.xml \ + && sed -i '301 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releases\n ' ./upgrade-acid/pom.xml \ + && sed -i '306 a \n aliyunmaven\n aliyun maven\n https://maven.aliyun.com/repository/spring-plugin\n \n ' ./upgrade-acid/pom.xml \ + && mvn package -DskipTests -Pdist -Dtar \ + && mkdir -p /root/output \ + && cp -r ./packaging/target/apache-hive-3.1.1-bin.tar.gz /root/output diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-hadoop-3.2.0 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-hadoop-3.2.0 new file mode 100644 index 0000000..8f1beb1 --- /dev/null +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-hadoop-3.2.0 @@ -0,0 +1,78 @@ +FROM arm64v8/openeuler:20.03-lts-sp1 + +# install base libs +WORKDIR /opt/tools/installed +RUN yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ + && mv /usr/bin/gcc /usr/bin/gcc-impl \ + && echo -e '#! /bin/sh\n/usr/bin/gcc-impl -fsigned-char "$@"' >> /usr/bin/gcc \ + && mv /usr/bin/g++ /usr/bin/g++-impl \ + && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ + && chmod +x /usr/bin/gcc /usr/bin/g++ \ + && yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen libtirpc-devel patch vim unzip diffutils.aarch64 cyrus-sasl* libtirpc-devel libXt-devel.aarch64 readline-devel.aarch64 autoconf cppunit-devel hostname + +# install maven and jdk +WORKDIR /opt/tools/installed + # download openjdk from github +RUN wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && tar -zxf OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/apache-maven-3.5.4-bin.tar.gz \ + #https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz + && tar -zxf apache-maven-3.5.4-bin.tar.gz -C /opt/tools/installed/ \ + && sed -i "146 a \n huaweimaven\n huawei maven\n https://mirrors.huaweicloud.com/repository/maven/\n central\n" /opt/tools/installed/apache-maven-3.5.4/conf/settings.xml + # set ENV +ENV JAVA_HOME=/opt/tools/installed/jdk8u282-b08 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.5.4 +ENV PATH=$MAVEN_HOME/bin:$PATH + +# install cmake 3.12.4 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/hadoop/cmake-3.12.4.tar.gz \ + # wget https://cmake.org/files/v3.12/cmake-3.12.4.tar.gz \ + && tar -zxf cmake-3.12.4.tar.gz \ + && cd cmake-3.12.4 \ + && ./bootstrap \ + && make -j8 \ + && make install + # && yum install -y openssl-devel zlib-devel automake libtool \ + +# install protocbuf 2.5.0 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/ENV/protobuf-2.5.0.tar.gz \ + # wget --no-check-certificate https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \ + && tar -xf protobuf-2.5.0.tar.gz \ + # && wget https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/bigdata/Patch/protoc.zip \ + && wget http://archive-server:50050/cache-data/BigData/ENV/protoc.zip \ + && unzip protoc.zip \ + && cp ./protoc/protoc.patch ./protobuf-2.5.0/src/google/protobuf/stubs/ \ + && cd protobuf-2.5.0/src/google/protobuf/stubs/ \ + && patch -p1 < protoc.patch \ + && cd /opt/tools/installed/protobuf-2.5.0 \ + && ./autogen.sh && ./configure CFLAGS='-fsigned-char' && make -j8 && make install \ + && mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 -Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc + +# install snappy 1.1.7 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/hadoop/1.1.7.tar.gz \ + # wget https://github.com/google/snappy/archive/1.1.7.tar.gz \ + && tar -zxf 1.1.7.tar.gz \ + && cd snappy-1.1.7/ \ + && sed -i "s/OFF/ON/g" /opt/tools/installed/snappy-1.1.7/CMakeLists.txt \ + && mkdir build \ + && cd build \ + && cmake ../ \ + && make install + +# install hadoop +WORKDIR /opt/tools/installed +# && yum -y install cyrus-sasl* libtirpc-devel\ +RUN wget http://archive-server:50050/cache-data/BigData/Apache/hadoop/hadoop-3.2.0-src.tar.gz \ + # wget https://archive.apache.org/dist/hadoop/common/hadoop-3.2.0/hadoop-3.2.0-src.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/Apache/hadoop/hadoop-3.2.0-openEuler.patch \ + && tar -xf hadoop-3.2.0-src.tar.gz \ + && mv hadoop-3.2.0-openEuler.patch hadoop-3.2.0-src \ + && cd hadoop-3.2.0-src \ + && patch -p1 < hadoop-3.2.0-openEuler.patch \ + && mvn package -DskipTests -Pdist,native -Dtar -Dsnappy.lib=/usr/local/lib64 -Dbundle.snappy -Dmaven.javadoc.skip=true \ + && mkdir -p /root/output \ + && cp hadoop-dist/target/hadoop-3.2.0.tar.gz /root/output diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-2.4.6 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-2.4.6 new file mode 100644 index 0000000..3fdc430 --- /dev/null +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-2.4.6 @@ -0,0 +1,56 @@ +FROM arm64v8/openeuler:20.03-lts-sp1 + +# install base libs +WORKDIR /opt/tools/installed +RUN yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ + && mv /usr/bin/gcc /usr/bin/gcc-impl \ + && echo -e '#! /bin/sh\n/usr/bin/gcc-impl -fsigned-char "$@"' >> /usr/bin/gcc \ + && mv /usr/bin/g++ /usr/bin/g++-impl \ + && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ + && chmod +x /usr/bin/gcc /usr/bin/g++ \ + && yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen libtirpc-devel patch vim unzip diffutils.aarch64 cyrus-sasl* libtirpc-devel libXt-devel.aarch64 readline-devel.aarch64 autoconf cppunit-devel hostname + +# install maven and jdk +WORKDIR /opt/tools/installed + # download openjdk from github +RUN wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && tar -zxf OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/apache-maven-3.5.4-bin.tar.gz \ + #https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz + && tar -zxf apache-maven-3.5.4-bin.tar.gz -C /opt/tools/installed/ \ + && sed -i "146 a \n huaweimaven\n huawei maven\n https://mirrors.huaweicloud.com/repository/maven/\n central\n" /opt/tools/installed/apache-maven-3.5.4/conf/settings.xml + # set ENV +ENV JAVA_HOME=/opt/tools/installed/jdk8u282-b08 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.5.4 +ENV PATH=$MAVEN_HOME/bin:$PATH + +# install R 3.1.1 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/R-3.1.1.tar.gz \ + # wget http://cran.rstudio.com/src/base/R-3/R-3.1.1.tar.gz \ + && tar -zxf R-3.1.1.tar.gz \ + && cd R-3.1.1 \ + && ./configure --enable-R-shlib --enable-R-static-lib --with-libpng --with-jpeglib --prefix=/opt/tools/installed/R-3.1.1 \ + && make all -j10 \ + && make install +ENV R_HOME=/opt/tools/installed/R-3.1.1 +ENV PATH=$R_HOME/bin:$PATH + +# install spark 2.4.6 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/spark-2.4.6.tgz \ + # wget https://archive.apache.org/dist/spark/spark-2.4.6/spark-2.4.6.tgz \ + && tar -xf spark-2.4.6.tgz \ + && cd spark-2.4.6 \ + && sed -i '263 a \n huaweicloudplugin\n huaweicloud plugin\n https://mirrors.huaweicloud.com/repository/maven/\n ' ./pom.xml \ + && sed -i '232 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ + && sed -i '237 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releses\n ' ./pom.xml \ + && sed -i '129,147d' dev/make-distribution.sh \ + && sed -i '129 a VERSION=2.4.6' dev/make-distribution.sh \ + && sed -i '130 a SCALA_VERSION=2.11' dev/make-distribution.sh \ + && sed -i '131 a SPARK_HADOOP_VERSION=2.6.5' dev/make-distribution.sh \ + && dev/make-distribution.sh --tgz -Pyarn,hive,hive-thriftserver,sparkr|tee buildlog \ + #&& build/mvn -DskipTests clean package |tee buildlog \ + && mkdir -p /root/output \ + && cp ./spark-2.4.6-bin-2.6.5.tgz /root/output diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-zookeeper-3.6.2 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-zookeeper-3.6.2 new file mode 100644 index 0000000..bcc64b4 --- /dev/null +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-zookeeper-3.6.2 @@ -0,0 +1,38 @@ +FROM arm64v8/openeuler:20.03-lts-sp1 + +# install base libs +WORKDIR /opt/tools/installed +RUN yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ + && mv /usr/bin/gcc /usr/bin/gcc-impl \ + && echo -e '#! /bin/sh\n/usr/bin/gcc-impl -fsigned-char "$@"' >> /usr/bin/gcc \ + && mv /usr/bin/g++ /usr/bin/g++-impl \ + && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ + && chmod +x /usr/bin/gcc /usr/bin/g++ \ + && yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen libtirpc-devel patch vim unzip diffutils.aarch64 cyrus-sasl* libtirpc-devel libXt-devel.aarch64 readline-devel.aarch64 autoconf cppunit-devel hostname + +# install maven and jdk +WORKDIR /opt/tools/installed + # download openjdk from github +RUN wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && tar -zxf OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/apache-maven-3.5.4-bin.tar.gz \ + #https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz + && tar -zxf apache-maven-3.5.4-bin.tar.gz -C /opt/tools/installed/ \ + && sed -i "146 a \n huaweimaven\n huawei maven\n https://mirrors.huaweicloud.com/repository/maven/\n central\n" /opt/tools/installed/apache-maven-3.5.4/conf/settings.xml + # set ENV +ENV JAVA_HOME=/opt/tools/installed/jdk8u282-b08 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.5.4 +ENV PATH=$MAVEN_HOME/bin:$PATH + +# install zookeeper 3.6.2 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/zookeeper/apache-zookeeper-3.6.2.tar.gz \ + # wget https://archive.apache.org/dist/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2.tar.gz \ + && tar -xf apache-zookeeper-3.6.2.tar.gz \ + && sed -i '960 a \n \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./apache-zookeeper-3.6.2/pom.xml \ + && sed -i '966 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releases\n \n ' ./apache-zookeeper-3.6.2/pom.xml \ + && cd apache-zookeeper-3.6.2 \ + && mvn clean install -DskipTests \ + && mkdir -p /root/output \ + && cp ./zookeeper-assembly/target/apache-zookeeper-3.6.2-bin.tar.gz /root/output -- Gitee From 3776d405dad7f93596118ba523a5c6d2edcef2b7 Mon Sep 17 00:00:00 2001 From: reflector1 Date: Fri, 30 Jul 2021 16:26:12 +0800 Subject: [PATCH 2/4] add openEuler apache spark-3.0.0 --- .../Apache/Dockerfile-spark-3.0.0 | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 new file mode 100644 index 0000000..8ed07b5 --- /dev/null +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 @@ -0,0 +1,56 @@ +FROM arm64v8/openeuler:20.03-lts-sp1 + +# install base libs +WORKDIR /opt/tools/installed +RUN yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ + && mv /usr/bin/gcc /usr/bin/gcc-impl \ + && echo -e '#! /bin/sh\n/usr/bin/gcc-impl -fsigned-char "$@"' >> /usr/bin/gcc \ + && mv /usr/bin/g++ /usr/bin/g++-impl \ + && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ + && chmod +x /usr/bin/gcc /usr/bin/g++ \ + && yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen libtirpc-devel patch vim unzip diffutils.aarch64 cyrus-sasl* libtirpc-devel libXt-devel.aarch64 readline-devel.aarch64 autoconf cppunit-devel hostname + +# install maven and jdk +WORKDIR /opt/tools/installed + # download openjdk from github +RUN wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && tar -zxf OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/apache-maven-3.6.3-bin.tar.gz \ + # wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz + && tar -zxf apache-maven-3.6.3-bin.tar.gz -C /opt/tools/installed/ \ + && sed -i "146 a \n huaweimaven\n huawei maven\n https://mirrors.huaweicloud.com/repository/maven/\n central\n" /opt/tools/installed/apache-maven-3.6.3/conf/settings.xml + # set ENV +ENV JAVA_HOME=/opt/tools/installed/jdk8u282-b08 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.6.3 +ENV PATH=$MAVEN_HOME/bin:$PATH + +# install R 3.1.1 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/R-3.1.1.tar.gz \ + # wget http://cran.rstudio.com/src/base/R-3/R-3.1.1.tar.gz \ + && tar -zxf R-3.1.1.tar.gz \ + && cd R-3.1.1 \ + && ./configure --enable-R-shlib --enable-R-static-lib --with-libpng --with-jpeglib --prefix=/opt/tools/installed/R-3.1.1 \ + && make all -j10 \ + && make install +ENV R_HOME=/opt/tools/installed/R-3.1.1 +ENV PATH=$R_HOME/bin:$PATH + +# install spark 3.0.0 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/v3.0.0.zip \ + # wget https://github.com/apache/spark/archive/refs/tags/v3.0.0.zip \ + && unzip v3.0.0.zip \ + && cd spark-3.0.0 \ + && sed -i '253 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ + && sed -i '258 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releses\n ' ./pom.xml \ + && sed -i '294 a \n huaweicloudplugin\n huaweicloud plugin\n https://mirrors.huaweicloud.com/repository/maven/\n ' ./pom.xml \ + && sed -i '162 s/x86/aarch/' common/network-yarn/pom.xml \ + && sed -i '163 s/x86/aarch/' common/network-yarn/pom.xml \ + && sed -i '164d' common/network-yarn/pom.xml \ + && sed -i '164d' common/network-yarn/pom.xml \ + && dev/make-distribution.sh --tgz -Pyarn,hive,hive-thriftserver,sparkr|tee buildlog \ + #&& build/mvn -DskipTests clean package |tee buildlog \ + && mkdir -p /root/output \ + && cp ./spark-3.0.0-bin-2.7.4.tgz /root/output -- Gitee From 592ce4933a9b82b65bd2c42aeaa6c83e672a8fce Mon Sep 17 00:00:00 2001 From: reflector1 Date: Mon, 2 Aug 2021 14:56:18 +0800 Subject: [PATCH 3/4] add openEuler apache spark-3.0.1 --- .../Apache/Dockerfile-spark-3.0.1 | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 new file mode 100644 index 0000000..a6ffd8e --- /dev/null +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 @@ -0,0 +1,57 @@ +FROM arm64v8/openeuler:20.03-lts-sp1 + +# install base libs +WORKDIR /opt/tools/installed +RUN yum -y install gcc.aarch64 gcc-c++.aarch64 gcc-gfortran.aarch64 libgcc.aarch64 \ + && mv /usr/bin/gcc /usr/bin/gcc-impl \ + && echo -e '#! /bin/sh\n/usr/bin/gcc-impl -fsigned-char "$@"' >> /usr/bin/gcc \ + && mv /usr/bin/g++ /usr/bin/g++-impl \ + && echo -e '#! /bin/sh\n/usr/bin/g++-impl -fsigned-char "$@"' >> /usr/bin/g++ \ + && chmod +x /usr/bin/gcc /usr/bin/g++ \ + && yum install -y boost.aarch64 boost-devel.aarch64 make wget openssl-devel zlib-devel automake libtool libstdc++-static glibc-static git snappy snappy-devel jansson-devel.aarch64 asciidoc.noarch doxygen libtirpc-devel patch vim unzip diffutils.aarch64 cyrus-sasl* libtirpc-devel libXt-devel.aarch64 readline-devel.aarch64 autoconf cppunit-devel hostname + +# install maven and jdk +WORKDIR /opt/tools/installed + # download openjdk from github +RUN wget http://archive-server:50050/cache-data/BigData/ENV/OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && tar -zxf OpenJDK8U-jdk_aarch64_linux_hotspot_jdk8u282-b08.tar.gz \ + && wget http://archive-server:50050/cache-data/BigData/ENV/apache-maven-3.6.3-bin.tar.gz \ + # wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz + && tar -zxf apache-maven-3.6.3-bin.tar.gz -C /opt/tools/installed/ \ + && sed -i "146 a \n huaweimaven\n huawei maven\n https://mirrors.huaweicloud.com/repository/maven/\n central\n" /opt/tools/installed/apache-maven-3.6.3/conf/settings.xml + # set ENV +ENV JAVA_HOME=/opt/tools/installed/jdk8u282-b08 +ENV PATH=$JAVA_HOME/bin:$PATH +ENV MAVEN_HOME=/opt/tools/installed/apache-maven-3.6.3 +ENV PATH=$MAVEN_HOME/bin:$PATH + +# install R 3.5.3 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/R-3.5.3.tar.gz \ + # wget http://cran.rstudio.com/src/base/R-3/R-3.5.3.tar.gz \ + && tar -zxf R-3.5.3.tar.gz \ + && cd R-3.5.3 \ + && yum install -y bzip2-devel xz-devel pcre-devel libcurl-devel \ + && ./configure --enable-R-shlib --enable-R-static-lib --with-libpng --with-jpeglib --prefix=/opt/tools/installed/R-3.5.3 \ + && make all -j10 \ + && make install +ENV R_HOME=/opt/tools/installed/R-3.5.3 +ENV PATH=$R_HOME/bin:$PATH + +# install spark 3.0.1 +WORKDIR /opt/tools/installed +RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/v3.0.1.zip \ + # wget https://github.com/apache/spark/archive/refs/tags/v3.0.1.zip \ + && unzip v3.0.1.zip \ + && cd spark-3.0.1 \ + && sed -i '261 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ + && sed -i '266 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releses\n ' ./pom.xml \ + && sed -i '302 a \n huaweicloudplugin\n huaweicloud plugin\n https://mirrors.huaweicloud.com/repository/maven/\n ' ./pom.xml \ + && sed -i '162 s/x86/aarch/' common/network-yarn/pom.xml \ + && sed -i '163 s/x86/aarch/' common/network-yarn/pom.xml \ + && sed -i '164d' common/network-yarn/pom.xml \ + && sed -i '164d' common/network-yarn/pom.xml \ + && dev/make-distribution.sh --tgz -Pyarn,hive,hive-thriftserver,sparkr|tee buildlog \ + #&& build/mvn -DskipTests clean package |tee buildlog \ + && mkdir -p /root/output \ + && cp ./spark-3.0.1-bin-2.7.4.tgz /root/output -- Gitee From 7f8dc3061733c41f053e040503730bbd0cb97184 Mon Sep 17 00:00:00 2001 From: reflector1 Date: Wed, 4 Aug 2021 10:51:02 +0800 Subject: [PATCH 4/4] update openEuler apache spark-3.0.* --- BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 | 3 ++- BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 index 8ed07b5..f352254 100644 --- a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.0 @@ -45,7 +45,8 @@ RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/v3.0.0.zip && cd spark-3.0.0 \ && sed -i '253 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ && sed -i '258 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releses\n ' ./pom.xml \ - && sed -i '294 a \n huaweicloudplugin\n huaweicloud plugin\n https://mirrors.huaweicloud.com/repository/maven/\n ' ./pom.xml \ + && sed -i '294 a \n kunpengcloudplugin\n kunpengcloud plugin\n https://mirrors.huaweicloud.com/kunpeng/maven/\n ' ./pom.xml \ + && sed -i '299 a \n huaweicloudplugin\n huaweicloud plugin\n https://mirrors.huaweicloud.com/repository/maven/\n ' ./pom.xml \ && sed -i '162 s/x86/aarch/' common/network-yarn/pom.xml \ && sed -i '163 s/x86/aarch/' common/network-yarn/pom.xml \ && sed -i '164d' common/network-yarn/pom.xml \ diff --git a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 index a6ffd8e..a0edf68 100644 --- a/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 +++ b/BigData/OpenEuler-20.03-lts-sp1/Apache/Dockerfile-spark-3.0.1 @@ -46,7 +46,8 @@ RUN wget http://archive-server:50050/cache-data/BigData/Apache/spark/v3.0.1.zip && cd spark-3.0.1 \ && sed -i '261 a \n kunpengmaven\n kunpeng maven\n https://mirrors.huaweicloud.com/kunpeng/maven\n ' ./pom.xml \ && sed -i '266 a \n hortonworksmaven\n hortonworks maven\n https://repo.hortonworks.com/content/repositories/releses\n ' ./pom.xml \ - && sed -i '302 a \n huaweicloudplugin\n huaweicloud plugin\n https://mirrors.huaweicloud.com/repository/maven/\n ' ./pom.xml \ + && sed -i '302 a \n kunpengcloudplugin\n kunpengcloud plugin\n https://mirrors.huaweicloud.com/kunpeng/maven/\n ' ./pom.xml \ + && sed -i '307 a \n huaweicloudplugin\n huaweicloud plugin\n https://mirrors.huaweicloud.com/repository/maven/\n ' ./pom.xml \ && sed -i '162 s/x86/aarch/' common/network-yarn/pom.xml \ && sed -i '163 s/x86/aarch/' common/network-yarn/pom.xml \ && sed -i '164d' common/network-yarn/pom.xml \ -- Gitee