diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/Dockerfile b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..95702093945564c296842aca84bd78ee4b2b4ef6 --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/Dockerfile @@ -0,0 +1,48 @@ +ARG BASE=openeuler/openeuler:24.03-lts-sp1 +FROM $BASE + +ARG VERSION=3.0.0 + +# install -y openjdk +COPY ssh/* /etc/ssh/ +RUN yum install -y java-11-openjdk-devel openssh openssh-clients sudo hostname && \ + yum clean all && \ + ssh-keygen -A && ssh-keygen -t ed25519 -P '' -f /root/.ssh/id_ed25519 && \ + cat /root/.ssh/id_ed25519.pub > /root/.ssh/authorized_keys && \ + chmod 0400 /root/.ssh/authorized_keys +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk + +# install zookeeper +RUN curl -fSL -o zookeeper.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.9.3/apache-zookeeper-3.9.3-bin.tar.gz; \ + mkdir -p /usr/local/zookeeper && \ + tar -zxf zookeeper.tar.gz -C /usr/local/zookeeper --strip-components=1 && \ + rm -rf zookeeper.tar.gz && \ + cd /usr/local/zookeeper/conf && \ + touch zoo.cfg && echo -e 'tickTime=2000\ndataDir=/var/lib/zookeeper\nclientPort=2181\nadmin.serverPort=8081' > zoo.cfg +ENV PATH=$PATH:/usr/local/zookeeper/bin +ENV ZOOKEEPER_HOME=/usr/local/zookeeper/ + +# install dependent hadoop +ARG HADOOP_VERSION=3.4.1 +RUN curl -fSL -o hadoop.tar.gz https://dlcdn.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz; \ + mkdir -p /usr/local/hadoop && \ + tar -zxf hadoop.tar.gz -C /usr/local/hadoop --strip-components=1 && \ + rm -rf hadoop.tar.gz +ENV PATH=$PATH:/usr/local/hadoop/bin +ENV HADOOP_HOME=/usr/local/hadoop/ +COPY hadoop/* $HADOOP_HOME/etc/hadoop/ + +# install accumulo +RUN curl -fSL -o accumulo.tar.gz https://dlcdn.apache.org/accumulo/${VERSION}/accumulo-${VERSION}-bin.tar.gz; \ + mkdir -p /usr/local/accumulo && \ + tar -zxf accumulo.tar.gz -C /usr/local/accumulo --strip-components=1 && \ + rm -rf accumulo.tar.gz +ENV PATH=$PATH:/usr/local/accumulo/bin +ENV ACCUMULO_HOME=/usr/local/accumulo/ +COPY properties/* $ACCUMULO_HOME/conf/ + +COPY entrypoint.sh / +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["init", "--upload-accumulo-props"] \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/entrypoint.sh b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..de9cb3b8462b9958eedb7508000d1a16679e38fe --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Start SSH service +/usr/sbin/sshd + +# Format HDFS +echo "Starting Zookeeper..." +"$ZOOKEEPER_HOME"/bin/zkServer.sh start + +# Format HDFS +echo "Formatting HDFS NameNode..." +hdfs namenode -format + +# Run Hadoop +echo "Starting Hadoop..." +$HADOOP_HOME/sbin/start-all.sh start +echo "Start History Server" +$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh --config $HADOOP_HOME/etc/hadoop start historyserver + +# Run accumulo with CMD +accumulo "$@" + +tail -f /dev/null \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/capacity-scheduler.xml b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/capacity-scheduler.xml new file mode 100644 index 0000000000000000000000000000000000000000..5eb4cddb0b3f079a35b06de37056322da06acaf7 --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/capacity-scheduler.xml @@ -0,0 +1,65 @@ + + + + + + yarn.scheduler.capacity.default.minimum-user-limit-percent + 100 + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.5 + + + yarn.scheduler.capacity.maximum-applications + 10000 + + + yarn.scheduler.capacity.node-locality-delay + 40 + + + yarn.scheduler.capacity.resource-calculator + org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator + + + yarn.scheduler.capacity.root.capacity + 100 + + + yarn.scheduler.capacity.root.default.capacity + 100 + + + yarn.scheduler.capacity.root.default.maximum-am-resource-percent + 0.5 + + + yarn.scheduler.capacity.root.default.maximum-capacity + 100 + + + yarn.scheduler.capacity.root.default.state + RUNNING + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + + yarn.scheduler.capacity.root.queues + default + + \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/core-site.xml b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/core-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..1821f54490e8bca27ecfef8c6589d123cd2d59b2 --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/core-site.xml @@ -0,0 +1,36 @@ + + + + + + + + + hadoop.tmp.dir + /data/hadoop + + + fs.defaultFS + hdfs://localhost:8020 + + + hadoop.http.staticuser.user + root + + + fs.hdfs.impl + org.apache.hadoop.hdfs.DistributedFileSystem + + \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/hadoop-env.sh b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/hadoop-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c57ab9c5ad5e25791831501351b207720a08d5a --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/hadoop-env.sh @@ -0,0 +1,8 @@ + +export JAVA_HOME=/usr/lib/jvm/java-11-openjdk +export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} +export HDFS_NAMENODE_USER="root" +export HDFS_DATANODE_USER="root" +export HDFS_SECONDARYNAMENODE_USER="root" +export YARN_RESOURCEMANAGER_USER="root" +export YARN_NODEMANAGER_USER="root" \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/hdfs-site.xml b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/hdfs-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..73d9f71ae7b0420f0e97436b8deb0a7dea72ff4d --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/hdfs-site.xml @@ -0,0 +1,24 @@ + + + + + + + + + dfs.replication + 1 + + \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/mapred-site.xml b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/mapred-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..1a9adfda2ec5d1e31ac9135195f69de972ea9084 --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/mapred-site.xml @@ -0,0 +1,28 @@ + + + + + + + + + mapreduce.framework.name + yarn + + + mapreduce.application.classpath + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/* + + \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/yarn-site.xml b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/yarn-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..5957ce2e16a277516fe5688bc581522e1aeff2e0 --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/hadoop/yarn-site.xml @@ -0,0 +1,61 @@ + + + + + + yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage + 98 + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.nodemanager.env-whitelist + JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME + + + yarn.resourcemanager.scheduler.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + yarn.log-aggregation-enable + true + + + yarn.log.dir + /data/hadoop + + + yarn.log.server.url + http://localhost:19888/jobhistory/logs + + + yarn.scheduler.minimum-allocation-vcores + 1 + + + yarn.scheduler.maximum-allocation-vcores + 1 + + + yarn.scheduler.minimum-allocation-mb + 256 + + + yarn.scheduler.maximum-allocation-mb + 4096 + + \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/accumulo.properties b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/accumulo.properties new file mode 100644 index 0000000000000000000000000000000000000000..2ba912a5bd1b9d4100b6cd73fe268a7a3d61e100 --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/accumulo.properties @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the main configuration file for Apache Accumulo. Available configuration properties can be +# found in the Accumulo documentation on the Accumulo project website (https://accumulo.apache.org/) +# Link for Accumulo 2.0: https://accumulo.apache.org/docs/2.0/administration/properties + +## Sets location in HDFS where Accumulo will store data +instance.volumes=hdfs://localhost:8020/accumulo + +## Sets location of Zookeepers +instance.zookeeper.host=localhost:2181 + +## Change secret before initialization. All Accumulo servers must have same secret +instance.secret=DEFAULT + +## Set to false if 'accumulo-util build-native' fails +tserver.memory.maps.native.enabled=true + +## Trace user +trace.user=root + +## Trace password +trace.password=secret + +## Search for available port if default is unavailable +tserver.port.search=true diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/log4j-monitor.properties b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/log4j-monitor.properties new file mode 100644 index 0000000000000000000000000000000000000000..2057ede4205ccd5cab6d3ce5f0c6da41e9dd51b1 --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/log4j-monitor.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Log4j 1.2 file that configures logging for Accumulo Monitor +## The system properties referenced below are configured by accumulo-env.sh + +## Define a console appender +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Target=System.out +log4j.appender.console.Threshold=ALL +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n +log4j.appender.console.layout=org.apache.log4j.PatternLayout + +## Define an appender for the Accumulo Monitor to log to its own web GUI +log4j.appender.gui=org.apache.accumulo.server.monitor.LogService +log4j.appender.gui.Threshold=WARN + +## Append monitor logs to its own web GUI +log4j.logger.org.apache.accumulo=INHERITED, gui + +## Append most logs to file +log4j.rootLogger=INFO, console diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/log4j-service.properties b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/log4j-service.properties new file mode 100644 index 0000000000000000000000000000000000000000..458804108e8f5342fa1dd92f8df2e554ad02bebf --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/properties/log4j-service.properties @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Log4j 1.2 file that configures logging for all Accumulo services (Master, TabletServer, GC, and Tracer) except Monitor +## The system properties referenced below are configured by accumulo-env.sh + +## Define a console appender +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Target=System.out +log4j.appender.console.Threshold=ALL +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n +log4j.appender.console.layout=org.apache.log4j.PatternLayout + +## Define an appender to send important logs to the the primary Accumulo Monitor +## The primary monitor is the one currently holding a shared lock in ZooKeeper, +## and is typically the one that started first. +log4j.appender.monitor=org.apache.accumulo.monitor.util.AccumuloMonitorAppender +log4j.appender.monitor.Threshold=WARN + +## Change this log level from OFF to one of the following to enable audit logging: +## INFO +## enables audit logging (inherit appenders from root logger) +## INFO, audit +## enables audit logging using the audit log appender +## (requires audit log file appender above to be uncommented) +log4j.logger.org.apache.accumulo.audit=OFF + +## Append logs to the primary Accumulo Monitor +log4j.logger.org.apache.accumulo=INHERITED, monitor + +## Constrain some particularly spammy loggers +log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO +log4j.logger.org.mortbay.log=WARN +log4j.logger.org.apache.zookeeper=ERROR + +## Append most logs to console +log4j.rootLogger=INFO, console diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/ssh/ssh_config b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/ssh/ssh_config new file mode 100644 index 0000000000000000000000000000000000000000..79611d11237584106f2f3a08c13a4659664afebc --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/ssh/ssh_config @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Host * + StrictHostKeyChecking no \ No newline at end of file diff --git a/Bigdata/accumulo/3.0.0/24.03-lts-sp1/ssh/sshd_config b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/ssh/sshd_config new file mode 100644 index 0000000000000000000000000000000000000000..8a7c77b72137a4ca5e2c8a83c704c55666791dce --- /dev/null +++ b/Bigdata/accumulo/3.0.0/24.03-lts-sp1/ssh/sshd_config @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ListenAddress 0.0.0.0 \ No newline at end of file diff --git a/Bigdata/accumulo/meta.yml b/Bigdata/accumulo/meta.yml index 2d6a7d665cffab80f3608e60c5fab8d1bea189fb..6698001f88fa37cbfb9a3d2235545c6a34efaa87 100644 --- a/Bigdata/accumulo/meta.yml +++ b/Bigdata/accumulo/meta.yml @@ -1,2 +1,4 @@ 2.1.3-oe2403sp1: - path: 2.1.3/24.03-lts-sp1/Dockerfile \ No newline at end of file + path: 2.1.3/24.03-lts-sp1/Dockerfile +3.0.0-oe2403sp1: + path: 3.0.0/24.03-lts-sp1/Dockerfile \ No newline at end of file