Skip to content

Commit 9e18d66

Browse files
satansonsatanson
committed
/home/hdfs
1 parent 7ca3dfa commit 9e18d66

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+540
-471
lines changed

‎Dockerfile-hadoop_debian-8.8

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,5 @@ RUN apt-get install sudo && rm -fr /var/lib/apt/lists/*
1717
RUN useradd -u 1000 -s /bin/bash -m hdfs
1818
RUN usermod -a -G sudo hdfs
1919
RUN echo hdfs ALL=(ALL) NOPASSWD : ALL > /etc/sudoers
20+
RUN apt-get update
21+
RUN apt-get install -y --no-install-recommends iptables && rm -rf /var/lib/apt/lists/*

‎EMP.sql

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
CREATE TABLE IF NOT EXISTS emp (
2+
id INTEGER NOT NULL PRIMARY KEY,
3+
name VARCHAR,
4+
age INTEGER,
5+
gender VARCHAR,
6+
boss VARCHAR,
7+
email VARCHAR
8+
);

‎bk_conf/bk_server.conf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,12 @@ bookiePort=3181
4343
#allowLoopback=false
4444

4545
# Directory Bookkeeper outputs its write ahead log
46-
journalDirectory=/root/bk_dat/journals
46+
journalDirectory=/home/hdfs/bk_dat/journals
4747

4848
# Directory Bookkeeper outputs ledger snapshots
4949
# could define multi directories to store snapshots, separated by ','
5050
# For example:
51-
ledgerDirectories=/root/bk_dat/ledgers
51+
ledgerDirectories=/home/hdfs/bk_dat/ledgers
5252
#
5353
# Ideally ledger dirs and journal dir are each in a differet device,
5454
# which reduce the contention between random i/o and sequential write.

‎create_rsgroup_system.hbase

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
create_namespace 'SYSTEM'
2-
create 'SYSTEM:test', 'f0', 'f1', 'f2'
2+
create 'SYSTEM:test', 'f0', 'f1', 'f2', SPLITS=>["user0", "user1", "user2","user3", "user4", "user5", "user6", "user7", "user8", "user9"]
33
put 'SYSTEM:test', 'r1', 'f0', 'foobar'
44
put 'SYSTEM:test', 'r2', 'f1', 'foobaz'
55
put 'SYSTEM:test', 'r3', 'f2', 'foobuzz'

‎datanode_conf/hdfs-site.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,12 @@
121121

122122
<property>
123123
<name>dfs.namenode.name.dir</name>
124-
<value>/root/hadoop_name_dir</value>
124+
<value>/home/hdfs/hadoop_name_dir</value>
125125
</property>
126126

127127
<property>
128128
<name>dfs.datanode.data.dir</name>
129-
<value>/root/hadoop_data_dir</value>
129+
<value>/home/hdfs/hadoop_data_dir</value>
130130
</property>
131131

132132
<property>

‎functions.sh

Lines changed: 110 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@ selectWord(){
33
local words=${1:?"undefined 'words'"};shift
44
local sep=${1:?"undefined 'sep'"};shift
55
local kw=${1:?"undefined 'kw'"};shift
6-
perl -e "print (join qq{\\t}, (grep /${kw}/, (split qq/${sep}/, qq/${words}/)))"
6+
perl -e "print (join qq{\\t}, (grep /^${kw}/, (split qq/${sep}/, qq/${words}/)))"
77
}
88

99
kill2docker(){
1010
local p=${1:?"undefined 'p'"};shift
1111
local kw=${1:?"undefined 'p'"};shift
12-
docker exec -it $p /bin/bash -c "ps -C java -o pid,cmd|perl -lne 'qx(kill -2 \$1) if /^\\s*\\b(\\d+)\\b.*${kw}/i'"
12+
docker kill $p
1313
}
1414

1515
killAll(){
@@ -20,3 +20,111 @@ killAll(){
2020
kill2docker $p ${kw}
2121
done
2222
}
23+
24+
selectOption(){
25+
test $# -gt 0
26+
select opt in $*;do
27+
echo ${opt}
28+
break;
29+
done
30+
}
31+
32+
selectHostList(){
33+
local dir=${1:?"missing 'dir'"};shift
34+
dir=${dir%%/}
35+
local oldshopt=$(set +o)
36+
set -e -o pipefail
37+
test -d ${dir}
38+
local n=$(ls ${dir}|wc -l)
39+
test ${n} -gt 0
40+
local selectList=$(ls ${dir}|xargs -i{} basename '{}')
41+
local chosed=""
42+
select opt in ${selectList};do
43+
chosed=${opt}
44+
break;
45+
done
46+
set +vx;eval "${oldshopt}"
47+
echo ${dir}/${chosed}
48+
}
49+
50+
confirm(){
51+
echo -n "Are your sure[yes/no]: "
52+
while : ; do
53+
read input
54+
input=$(perl -e "print qq/\L${input}\E/")
55+
case ${input} in
56+
y|ye|yes)
57+
break
58+
;;
59+
n|no)
60+
echo "operation is cancelled!!!"
61+
exit 0
62+
;;
63+
*)
64+
echo -n "invalid choice, choose again!!! [yes|no]: "
65+
;;
66+
esac
67+
done
68+
}
69+
70+
checkArgument(){
71+
local name=${1:?"missing 'name'"};shift
72+
local arg=${1:?"missing 'arg'"};shift
73+
local alternatives=${1:?"missing 'alternatives'"};shift
74+
75+
if [ -z ${alternatives} ];then
76+
echo "ERROR: empty alternatives for '${name}', value='${arg}'" >&2
77+
exit 1
78+
fi
79+
80+
if test x$(perl -e "print qq/${alternatives}/=~/^\w+(?:\|\w+)*$/")x != x1x;then
81+
echo "ERROR: alternatives must be in format word1|word2|word3..., name='${name}', value='${arg}', alternatives='${alternatives}" >&2
82+
exit 2
83+
fi
84+
85+
if test x$(perl -e "print qq/$arg/=~/^(?:${alternatives})$/")x != x1x; then
86+
echo "ERROR: unmatched argument, name='${name}', value='${arg}', alternatives='${alternatives}'" >&2
87+
exit 1
88+
fi
89+
}
90+
91+
isIn(){
92+
local arg=${1:?"missing 'arg'"};shift
93+
local alternatives=${1:?"missing 'alternatives'"};shift
94+
95+
if [ -z ${alternatives} ];then
96+
echo "ERROR: empty alternatives, value=${arg}" >&2
97+
exit 1
98+
fi
99+
100+
if test x$(perl -e "print qq/${alternatives}/=~/^\w+(?:\|\w+)*$/")x != x1x;then
101+
echo "ERROR: alternatives must be in format word1|word2|word3..., value='${arg}', alternatives='${alternatives}" >&2
102+
exit 2
103+
fi
104+
105+
if test x$(perl -e "print qq/$arg/=~/^(?:${alternatives})$/")x != x1x; then
106+
return 1
107+
else
108+
return 0
109+
fi
110+
}
111+
112+
startsWith(){
113+
local arg=${1:?"missing 'arg'"};shift
114+
local prefix=${1:?"missing 'prefix'"};shift
115+
if [ "x${arg##${prefix}}x" = "x${arg}x" ];then
116+
return 1
117+
else
118+
return 0
119+
fi
120+
}
121+
122+
endsWith(){
123+
local arg=${1:?"missing 'arg'"};shift
124+
local suffix=${1:?"missing 'prefix'"};shift
125+
if [ "x${arg%%${suffix}}x" = "x${arg}x" ];then
126+
return 1
127+
else
128+
return 0
129+
fi
130+
}

‎get-schema.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
t=$1
4+
echo "!desc $t;" > $t.1
5+
./sqlline | sed 's#\[1;32m | \[m##g' | sed '/---/d' | sed '/TABLE_CAT/d' | sed 's#[1;32m##g' | sed /d'| sed 's#|##g' | awk '{if ($0 ~"PK ") {if (NR != 1) print ") compression = '\''snappy'\'', UPDATE_CACHE_FREQUENCY=30000, COLUMN_ENCODED_BYTES=0;"; print "create table if not exists "$2"."$3" (\n\"PK\" VARCHAR PRIMARY KEY"; } else {print ",\""$4"\" ", $6}; }''END{print ") compression = '\''snappy'\'', UPDATE_CACHE_FREQUENCY=30000, COLUMN_ENCODED_BYTES=0;"}' | sed 's#[m##g' > $t.sql
6+
rm $t.1

‎hadoop_conf_client/hadoop-env.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPT
4949
export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS \
5050
-Dcom.sun.management.jmxremote.port=10004 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
5151

52-
export HDFS_NAMENODE_USER=root
53-
export HDFS_SECONDARYNAMENODE_USER=root
54-
export HDFS_DATANODE_USER=root
52+
export HDFS_NAMENODE_USER=hdfs
53+
export HDFS_SECONDARYNAMENODE_USER=hdfs
54+
export HDFS_DATANODE_USER=hdfs
5555
# Therefore, the vast majority (BUT NOT ALL!) of these defaults
5656
# are configured for substitution and not append. If append
5757
# is preferable, modify this file accordingly.

‎hadoop_conf_client/hbase-site.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
<property>
3030
<name>hbase.tmp.dir</name>
31-
<value>/root/hbase_tmp</value>
31+
<value>/home/hdfs/hbase_tmp</value>
3232
</property>
3333

3434
<property>

‎hadoop_conf_client/hdfs-site.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,12 +97,12 @@
9797

9898
<property>
9999
<name>dfs.namenode.name.dir</name>
100-
<value>/root/hadoop_name_dir</value>
100+
<value>/home/hdfs/hadoop_name_dir</value>
101101
</property>
102102

103103
<property>
104104
<name>dfs.datanode.data.dir</name>
105-
<value>/root/hadoop_data_dir</value>
105+
<value>/home/hdfs/hadoop_data_dir</value>
106106
</property>
107107

108108
<property>

0 commit comments

Comments
 (0)