Skip to content

Commit e190a9b

Browse files
satansonsatanson
committed
by grakra
1 parent 9e18d66 commit e190a9b

File tree

16 files changed

+155
-9
lines changed

16 files changed

+155
-9
lines changed

‎export.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
./hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot \
3+
-snapshot "SYSTEM.test.snapshot.2019.08.30" \
4+
-copy-from hdfs://grakrabackend/hbase \
5+
-copy-to hdfs://grakrabackend/hbase2 \
6+
-mappers 5 \
7+
-bandwidth 10 \
8+
-overwrite \
9+
2>&1 |tee SYSTEM.test.snapshot.2019.08.30.log

‎export2.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
./hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot \
3+
-snapshot "SYSTEM.test.snapshot.2019.08.30" \
4+
-copy-from hdfs://grakrabackend/hbase2 \
5+
-copy-to hdfs://grakrabackend/hbase \
6+
-mappers 1 \
7+
-bandwidth 1 \
8+
-overwrite \
9+
2>&1 |tee SYSTEM.test.snapshot.2019.08.30.log

‎start_hdfs.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ startNode(){
2929
-v ${PWD}/${name}_dat:${targetDataDir}
3030
-v ${PWD}/${name}_logs:/home/hdfs/hadoop/logs
3131
-v ${confDir}:/home/hdfs/hadoop/etc/hadoop
32+
-v ${PWD}/start_hdfs_node.sh:/home/hdfs/hadoop/start_hdfs_node.sh
3233
"
33-
docker run $flags hadoop_debian:8.8 /home/hdfs/hadoop/bin/hdfs ${command}
34+
docker run $flags hadoop_debian:8.8 /home/hdfs/hadoop/start_hdfs_node.sh ${command}
3435
}
3536

3637
format(){

‎start_hdfs_node.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ if [ "x${command}x" != "xnamenodex" -a "x${command}x" != "xdatanodex" ];then
1010
exit 1
1111
fi
1212

13-
base_dir=/root/hadoop
14-
cd $base_dir
13+
basedir=/home/hdfs/hadoop
14+
cd $basedir
1515

1616
bin=`cd bin; pwd`
1717

‎start_mr_history_server.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
set -e -o pipefail
33

44
for node in yarnrm{0..1} yarnnm{0..5};do
5-
docker exec -it ${node} /root/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver
5+
docker exec -it ${node} /home/hdfs/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver
66
done

‎start_yarn.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ startNode(){
3535
-v ${PWD}/${name}_dat:${targetDataDir}
3636
-v ${PWD}/${name}_log:/home/hdfs/hadoop/logs
3737
-v ${confDir}:/home/hdfs/hadoop/etc/hadoop
38+
-v ${PWD}/start_yarn_node.sh:/home/hdfs/hadoop/start_yarn_node.sh
3839
"
39-
docker run $flags hadoop_debian:8.8 /home/hdfs/hadoop/bin/yarn ${command}
40+
docker run $flags hadoop_debian:8.8 /home/hdfs/hadoop/start_yarn_node.sh ${command}
4041
}
4142

4243
start_yarnrm(){

‎start_yarn_node.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ if [ "x${command}x" != "xresourcemanagerx" -a "x${command}x" != "xnodemanagerx"
99
exit 1
1010
fi
1111

12-
base_dir=/root/hadoop
13-
USER=root
12+
basedir=/home/hdfs/hadoop
13+
USER=hdfs
1414

15-
cd $base_dir
15+
cd $basedir
1616

1717
bin=`cd bin; pwd`
1818

‎start_zk.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ zkRoot=${basedir}/../hadoop_all/zookeeper
66

77
set -e -o pipefail
88
if [ -n "${bootstrap}" ];then
9-
sudo rm -fr ${basedir:?"undefined"}/zk*_data/*
9+
rm -fr ${basedir:?"undefined"}/zk*_data/*
1010
fi
1111

1212
zkNum=3

‎yarnnm0_conf/yarn-site.xml

+17
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@
176176
<value>604800</value>
177177
</property>
178178

179+
<property>
180+
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
181+
<value>99.9</value>
182+
</property>
183+
179184
<property>
180185
<name>yarn.nodemanager.aux-services</name>
181186
<value>mapreduce_shuffle,spark_shuffle</value>
@@ -189,6 +194,18 @@
189194
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190195
</property>
191196

197+
<property>
198+
<name>yarn.nodemanager.vmem-check-enabled</name>
199+
<value>false</value>
200+
<description>Whether virtual memory limits will be enforced for containers</description>
201+
</property>
202+
203+
<property>
204+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
205+
<value>4</value>
206+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
207+
</property>
208+
192209
<property>
193210
<name>yarn.log.server.url</name>
194211
<value>http://yarnnm0:19888/jobhistory/logs/</value>

‎yarnnm1_conf/yarn-site.xml

+17
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@
176176
<value>604800</value>
177177
</property>
178178

179+
<property>
180+
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
181+
<value>99.9</value>
182+
</property>
183+
179184
<property>
180185
<name>yarn.nodemanager.aux-services</name>
181186
<value>mapreduce_shuffle,spark_shuffle</value>
@@ -189,6 +194,18 @@
189194
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190195
</property>
191196

197+
<property>
198+
<name>yarn.nodemanager.vmem-check-enabled</name>
199+
<value>false</value>
200+
<description>Whether virtual memory limits will be enforced for containers</description>
201+
</property>
202+
203+
<property>
204+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
205+
<value>4</value>
206+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
207+
</property>
208+
192209
<property>
193210
<name>yarn.log.server.url</name>
194211
<value>http://yarnnm1:19888/jobhistory/logs/</value>

‎yarnnm2_conf/yarn-site.xml

+17
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@
176176
<value>604800</value>
177177
</property>
178178

179+
<property>
180+
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
181+
<value>99.9</value>
182+
</property>
183+
179184
<property>
180185
<name>yarn.nodemanager.aux-services</name>
181186
<value>mapreduce_shuffle,spark_shuffle</value>
@@ -189,6 +194,18 @@
189194
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190195
</property>
191196

197+
<property>
198+
<name>yarn.nodemanager.vmem-check-enabled</name>
199+
<value>false</value>
200+
<description>Whether virtual memory limits will be enforced for containers</description>
201+
</property>
202+
203+
<property>
204+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
205+
<value>4</value>
206+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
207+
</property>
208+
192209
<property>
193210
<name>yarn.log.server.url</name>
194211
<value>http://yarnnm2:19888/jobhistory/logs/</value>

‎yarnnm3_conf/yarn-site.xml

+17
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@
176176
<value>604800</value>
177177
</property>
178178

179+
<property>
180+
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
181+
<value>99.9</value>
182+
</property>
183+
179184
<property>
180185
<name>yarn.nodemanager.aux-services</name>
181186
<value>mapreduce_shuffle,spark_shuffle</value>
@@ -189,6 +194,18 @@
189194
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190195
</property>
191196

197+
<property>
198+
<name>yarn.nodemanager.vmem-check-enabled</name>
199+
<value>false</value>
200+
<description>Whether virtual memory limits will be enforced for containers</description>
201+
</property>
202+
203+
<property>
204+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
205+
<value>4</value>
206+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
207+
</property>
208+
192209
<property>
193210
<name>yarn.log.server.url</name>
194211
<value>http://yarnnm3:19888/jobhistory/logs/</value>

‎yarnnm4_conf/yarn-site.xml

+17
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@
176176
<value>604800</value>
177177
</property>
178178

179+
<property>
180+
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
181+
<value>99.9</value>
182+
</property>
183+
179184
<property>
180185
<name>yarn.nodemanager.aux-services</name>
181186
<value>mapreduce_shuffle,spark_shuffle</value>
@@ -189,6 +194,18 @@
189194
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190195
</property>
191196

197+
<property>
198+
<name>yarn.nodemanager.vmem-check-enabled</name>
199+
<value>false</value>
200+
<description>Whether virtual memory limits will be enforced for containers</description>
201+
</property>
202+
203+
<property>
204+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
205+
<value>4</value>
206+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
207+
</property>
208+
192209
<property>
193210
<name>yarn.log.server.url</name>
194211
<value>http://yarnnm4:19888/jobhistory/logs/</value>

‎yarnnm5_conf/yarn-site.xml

+17
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,11 @@
176176
<value>604800</value>
177177
</property>
178178

179+
<property>
180+
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
181+
<value>99.9</value>
182+
</property>
183+
179184
<property>
180185
<name>yarn.nodemanager.aux-services</name>
181186
<value>mapreduce_shuffle,spark_shuffle</value>
@@ -189,6 +194,18 @@
189194
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190195
</property>
191196

197+
<property>
198+
<name>yarn.nodemanager.vmem-check-enabled</name>
199+
<value>false</value>
200+
<description>Whether virtual memory limits will be enforced for containers</description>
201+
</property>
202+
203+
<property>
204+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
205+
<value>4</value>
206+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
207+
</property>
208+
192209
<property>
193210
<name>yarn.log.server.url</name>
194211
<value>http://yarnnm5:19888/jobhistory/logs/</value>

‎yarnrm0_conf/yarn-site.xml

+12
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,18 @@
189189
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190190
</property>
191191

192+
<property>
193+
<name>yarn.nodemanager.vmem-check-enabled</name>
194+
<value>false</value>
195+
<description>Whether virtual memory limits will be enforced for containers</description>
196+
</property>
197+
198+
<property>
199+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
200+
<value>4</value>
201+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
202+
</property>
203+
192204
<property>
193205
<name>yarn.log.server.url</name>
194206
<value>http://yarnrm0:19888/jobhistory/logs/</value>

‎yarnrm1_conf/yarn-site.xml

+12
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,18 @@
189189
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
190190
</property>
191191

192+
<property>
193+
<name>yarn.nodemanager.vmem-check-enabled</name>
194+
<value>false</value>
195+
<description>Whether virtual memory limits will be enforced for containers</description>
196+
</property>
197+
198+
<property>
199+
<name>yarn.nodemanager.vmem-pmem-ratio</name>
200+
<value>4</value>
201+
<description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
202+
</property>
203+
192204
<property>
193205
<name>yarn.log.server.url</name>
194206
<value>http://yarnrm1:19888/jobhistory/logs/</value>

0 commit comments

Comments
 (0)