diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/Dockerfile b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..fd1c538150d278647c94cf398eaa725719040585 --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/Dockerfile @@ -0,0 +1,25 @@ +ARG BASE=openeuler/openeuler:24.03-lts-sp1 +FROM $BASE + +ARG VERSION=3.4.2 + +COPY ssh/* /etc/ssh/ +COPY entrypoint.sh / +RUN yum install -y java-11-openjdk-devel openssh openssh-clients sudo hostname && \ + yum clean all && \ + ssh-keygen -A && ssh-keygen -t ed25519 -P '' -f /root/.ssh/id_ed25519 && \ + cat /root/.ssh/id_ed25519.pub > /root/.ssh/authorized_keys && \ + chmod 0400 /root/.ssh/authorized_keys && \ + chmod +x /entrypoint.sh + +RUN curl -fSL -o hadoop.tar.gz https://dlcdn.apache.org/hadoop/common/hadoop-${VERSION}/hadoop-${VERSION}.tar.gz; \ + mkdir -p /usr/local/hadoop && \ + tar -zxf hadoop.tar.gz -C /usr/local/hadoop --strip-components=1 && \ + rm -rf hadoop.tar.gz +ENV PATH=$PATH:/usr/local/hadoop/bin +ENV HADOOP_HOME=/usr/local/hadoop +COPY hadoop/* $HADOOP_HOME/etc/hadoop/ + +EXPOSE 9870 8088 19888 50070 50075 50010 50020 50090 +ENTRYPOINT [ "/entrypoint.sh" ] +CMD [ "start" ] \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/entrypoint.sh b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..8f5400ad04ccbd32712e4966b3a0d141f16eb01d --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Start SSH service +/usr/sbin/sshd + +# Format HDFS +echo "Formatting HDFS NameNode..." +hdfs namenode -format + +# Run Hadoop with CMD arguments +$HADOOP_HOME/sbin/start-all.sh "$@" + +echo "Start History Server" +$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh --config $HADOOP_HOME/etc/hadoop start historyserver + +tail -f /dev/null \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/capacity-scheduler.xml b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/capacity-scheduler.xml new file mode 100644 index 0000000000000000000000000000000000000000..5eb4cddb0b3f079a35b06de37056322da06acaf7 --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/capacity-scheduler.xml @@ -0,0 +1,65 @@ + + + + + + yarn.scheduler.capacity.default.minimum-user-limit-percent + 100 + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.5 + + + yarn.scheduler.capacity.maximum-applications + 10000 + + + yarn.scheduler.capacity.node-locality-delay + 40 + + + yarn.scheduler.capacity.resource-calculator + org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator + + + yarn.scheduler.capacity.root.capacity + 100 + + + yarn.scheduler.capacity.root.default.capacity + 100 + + + yarn.scheduler.capacity.root.default.maximum-am-resource-percent + 0.5 + + + yarn.scheduler.capacity.root.default.maximum-capacity + 100 + + + yarn.scheduler.capacity.root.default.state + RUNNING + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + + yarn.scheduler.capacity.root.queues + default + + \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/core-site.xml b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/core-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..2be9b2429a1800d41801b8e5cfbc7b7d26c23b4a --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/core-site.xml @@ -0,0 +1,36 @@ + + + + + + + + + hadoop.tmp.dir + /data/hadoop + + + fs.defaultFS + hdfs://localhost:9000 + + + hadoop.http.staticuser.user + root + + + fs.hdfs.impl + org.apache.hadoop.hdfs.DistributedFileSystem + + \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/hadoop-env.sh b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/hadoop-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c57ab9c5ad5e25791831501351b207720a08d5a --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/hadoop-env.sh @@ -0,0 +1,8 @@ + +export JAVA_HOME=/usr/lib/jvm/java-11-openjdk +export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} +export HDFS_NAMENODE_USER="root" +export HDFS_DATANODE_USER="root" +export HDFS_SECONDARYNAMENODE_USER="root" +export YARN_RESOURCEMANAGER_USER="root" +export YARN_NODEMANAGER_USER="root" \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/hdfs-site.xml b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/hdfs-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..73d9f71ae7b0420f0e97436b8deb0a7dea72ff4d --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/hdfs-site.xml @@ -0,0 +1,24 @@ + + + + + + + + + dfs.replication + 1 + + \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/mapred-site.xml b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/mapred-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..1a9adfda2ec5d1e31ac9135195f69de972ea9084 --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/mapred-site.xml @@ -0,0 +1,28 @@ + + + + + + + + + mapreduce.framework.name + yarn + + + mapreduce.application.classpath + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/* + + \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/yarn-site.xml b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/yarn-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..5957ce2e16a277516fe5688bc581522e1aeff2e0 --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/hadoop/yarn-site.xml @@ -0,0 +1,61 @@ + + + + + + yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage + 98 + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.nodemanager.env-whitelist + JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME + + + yarn.resourcemanager.scheduler.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + yarn.log-aggregation-enable + true + + + yarn.log.dir + /data/hadoop + + + yarn.log.server.url + http://localhost:19888/jobhistory/logs + + + yarn.scheduler.minimum-allocation-vcores + 1 + + + yarn.scheduler.maximum-allocation-vcores + 1 + + + yarn.scheduler.minimum-allocation-mb + 256 + + + yarn.scheduler.maximum-allocation-mb + 4096 + + \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/ssh/ssh_config b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/ssh/ssh_config new file mode 100644 index 0000000000000000000000000000000000000000..79611d11237584106f2f3a08c13a4659664afebc --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/ssh/ssh_config @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Host * + StrictHostKeyChecking no \ No newline at end of file diff --git a/Bigdata/hadoop/3.4.2/24.03-lts-sp1/ssh/sshd_config b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/ssh/sshd_config new file mode 100644 index 0000000000000000000000000000000000000000..8a7c77b72137a4ca5e2c8a83c704c55666791dce --- /dev/null +++ b/Bigdata/hadoop/3.4.2/24.03-lts-sp1/ssh/sshd_config @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ListenAddress 0.0.0.0 \ No newline at end of file diff --git a/Bigdata/hadoop/README.md b/Bigdata/hadoop/README.md index b9f73104b4cc7e6fc55dca0a16d85588dbd44074..5a34d0863491f9633b6ff7c9d8178139787d5bd3 100644 --- a/Bigdata/hadoop/README.md +++ b/Bigdata/hadoop/README.md @@ -16,6 +16,7 @@ Learn more on [hadoop website](https://hadoop.apache.org/). The tag of each hadoop docker image is consist of the version of hadoop and the version of basic image. The details are as follows | Tags | Currently | Architectures| |--|--|--| +|[3.4.2-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/hadoop/3.4.2/24.03-lts-sp1/Dockerfile) | hadoop 3.4.2 on openEuler 24.03-LTS-SP1 | amd64, arm64 | |[3.4.1-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/hadoop/3.4.1/24.03-lts-sp1/Dockerfile)| Apache hadoop 3.4.1 on openEuler 24.03-LTS-SP1 | amd64, arm64 | # Usage diff --git a/Bigdata/hadoop/doc/image-info.yml b/Bigdata/hadoop/doc/image-info.yml index aea20c1896e4d04d28dd5a86680abc329228cddd..40be0283ca7357d54caf2a2b07d9f431a4a3a00e 100644 --- a/Bigdata/hadoop/doc/image-info.yml +++ b/Bigdata/hadoop/doc/image-info.yml @@ -11,6 +11,7 @@ tags: | | Tags | Currently | Architectures| |------|-----------|---------------| + |[3.4.2-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/hadoop/3.4.2/24.03-lts-sp1/Dockerfile) | hadoop 3.4.2 on openEuler 24.03-LTS-SP1 | amd64, arm64 | |[3.4.1-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/hadoop/3.4.1/24.03-lts-sp1/Dockerfile)| Apache hadoop 3.4.1 on openEuler 24.03-LTS-SP1 | amd64, arm64 | download: | diff --git a/Bigdata/hadoop/meta.yml b/Bigdata/hadoop/meta.yml index e9bc4ed820d84652e4408ad364865bf9fd4953d3..2c4240e80e93ea151bd5a97b5754a56a9184b52f 100644 --- a/Bigdata/hadoop/meta.yml +++ b/Bigdata/hadoop/meta.yml @@ -1,2 +1,4 @@ 3.4.1-oe2403sp1: - path: 3.4.1/24.03-lts-sp1/Dockerfile \ No newline at end of file + path: 3.4.1/24.03-lts-sp1/Dockerfile +3.4.2-oe2403sp1: + path: 3.4.2/24.03-lts-sp1/Dockerfile \ No newline at end of file