Skip to content

Commit 4d88f98

Browse files
author
drankye
committed
HADOOP-19085. Compatibility Benchmark over HCFS Implementations
Contributed by Han Liu
1 parent 783cc3e commit 4d88f98

File tree

66 files changed

+6127
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+6127
-0
lines changed

‎hadoop-tools/hadoop-compat-bench/pom.xml

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
16+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
17+
<modelVersion>4.0.0</modelVersion>
18+
<parent>
19+
<groupId>org.apache.hadoop</groupId>
20+
<artifactId>hadoop-project</artifactId>
21+
<version>3.5.0-SNAPSHOT</version>
22+
<relativePath>../../hadoop-project</relativePath>
23+
</parent>
24+
<artifactId>hadoop-compat-bench</artifactId>
25+
<version>3.5.0-SNAPSHOT</version>
26+
<packaging>jar</packaging>
27+
28+
<description>Apache Hadoop Compatibility</description>
29+
<name>Apache Hadoop Compatibility Benchmark</name>
30+
31+
<dependencies>
32+
<dependency>
33+
<groupId>org.apache.hadoop</groupId>
34+
<artifactId>hadoop-common</artifactId>
35+
<scope>provided</scope>
36+
</dependency>
37+
<dependency>
38+
<!-- Should we keep this -->
39+
<groupId>org.apache.hadoop</groupId>
40+
<artifactId>hadoop-hdfs</artifactId>
41+
<scope>provided</scope>
42+
</dependency>
43+
<dependency>
44+
<groupId>junit</groupId>
45+
<artifactId>junit</artifactId>
46+
<scope>compile</scope>
47+
</dependency>
48+
49+
<!-- For test -->
50+
<dependency>
51+
<groupId>org.apache.hadoop</groupId>
52+
<artifactId>hadoop-hdfs-client</artifactId>
53+
<scope>test</scope>
54+
</dependency>
55+
<dependency>
56+
<groupId>org.apache.hadoop</groupId>
57+
<artifactId>hadoop-common</artifactId>
58+
<type>test-jar</type>
59+
<scope>test</scope>
60+
</dependency>
61+
<dependency>
62+
<groupId>org.apache.hadoop</groupId>
63+
<artifactId>hadoop-hdfs</artifactId>
64+
<type>test-jar</type>
65+
<scope>test</scope>
66+
</dependency>
67+
<dependency>
68+
<groupId>org.mockito</groupId>
69+
<artifactId>mockito-core</artifactId>
70+
<scope>test</scope>
71+
</dependency>
72+
</dependencies>
73+
74+
<build>
75+
<plugins>
76+
<plugin>
77+
<groupId>org.apache.maven.plugins</groupId>
78+
<artifactId>maven-jar-plugin</artifactId>
79+
<configuration>
80+
<archive>
81+
<manifest>
82+
<mainClass>org.apache.hadoop.fs.compat.HdfsCompatTool</mainClass>
83+
</manifest>
84+
</archive>
85+
</configuration>
86+
<executions>
87+
<execution>
88+
<goals>
89+
<goal>test-jar</goal>
90+
</goals>
91+
<configuration>
92+
<archive>
93+
<manifest>
94+
<mainClass>org.apache.hadoop.fs.compat.hdfs.HdfsCompatMiniCluster</mainClass>
95+
</manifest>
96+
</archive>
97+
</configuration>
98+
</execution>
99+
</executions>
100+
</plugin>
101+
<plugin>
102+
<groupId>org.apache.maven.plugins</groupId>
103+
<artifactId>maven-surefire-plugin</artifactId>
104+
<configuration>
105+
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
106+
</configuration>
107+
</plugin>
108+
</plugins>
109+
<resources>
110+
<resource>
111+
<directory>src/main/resources</directory>
112+
</resource>
113+
<resource>
114+
<directory>shell</directory>
115+
</resource>
116+
</resources>
117+
</build>
118+
</project>

‎hadoop-tools/hadoop-compat-bench/shell/cases/attr.t

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/sh
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
. $(dirname "$0")/../misc.sh
19+
20+
echo "Hello World!" > "${localDir}/dat"
21+
hadoop fs -put "${localDir}/dat" "${baseDir}/"
22+
23+
echo "1..10"
24+
25+
# 1. chown
26+
hadoop fs -chown "hadoop-compat-bench-user" "${baseDir}/dat"
27+
expect_out "chown" "user:hadoop-compat-bench-user" hadoop fs -stat "user:%u" "${baseDir}/dat"
28+
29+
# 2. chgrp
30+
hadoop fs -chgrp "hadoop-compat-bench-group" "${baseDir}/dat"
31+
expect_out "chgrp" "group:hadoop-compat-bench-group" hadoop fs -stat "group:%g" "${baseDir}/dat"
32+
33+
# 3. chmod
34+
hadoop fs -chmod 777 "${baseDir}/dat"
35+
expect_out "chmod" "perm:777" hadoop fs -stat "perm:%a" "${baseDir}/dat"
36+
37+
# 4. touch
38+
hadoop fs -touch -m -t "20000615:000000" "${baseDir}/dat"
39+
expect_out "touch" "date:2000-06-.*" hadoop fs -stat "date:%y" "${baseDir}/dat"
40+
41+
# 5. setfattr
42+
expect_ret "setfattr" 0 hadoop fs -setfattr -n "user.key" -v "value" "${baseDir}/dat"
43+
44+
# 6. getfattr
45+
expect_out "getfattr" ".*value.*" hadoop fs -getfattr -n "user.key" "${baseDir}/dat"
46+
47+
# 7. setfacl
48+
expect_ret "setfacl" 0 hadoop fs -setfacl -m "user:foo:---" "${baseDir}/dat"
49+
50+
# 8. getfacl
51+
expect_out "getfacl" ".*foo.*" hadoop fs -getfacl "${baseDir}/dat"
52+
53+
# 9. setrep
54+
hadoop fs -setrep 1 "${baseDir}/dat"
55+
expect_out "setrep" "replication:1" hadoop fs -stat "replication:%r" "${baseDir}/dat"
56+
57+
# 10. checksum
58+
expect_ret "checksum" 0 hadoop fs -checksum "${baseDir}/dat" # TODO

‎hadoop-tools/hadoop-compat-bench/shell/cases/concat.t

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/sh
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
. $(dirname "$0")/../misc.sh
19+
20+
echo "Hello World!" > "${localDir}/dat"
21+
hadoop fs -put "${localDir}/dat" "${baseDir}/src1"
22+
hadoop fs -put "${localDir}/dat" "${baseDir}/src2"
23+
24+
echo "1..3"
25+
26+
# 1. touchz
27+
hadoop fs -touchz "${baseDir}/dat"
28+
expect_out "touchz" "size:0" hadoop fs -stat "size:%b" "${baseDir}/dat"
29+
30+
# 2. concat
31+
expect_ret "concat" 0 hadoop fs -concat "${baseDir}/dat" "${baseDir}/src1" "${baseDir}/src2"
32+
# expect_out "size:26" hadoop fs -stat "size:%b" "${baseDir}/dat"
33+
34+
# 3. getmerge
35+
hadoop fs -getmerge "${baseDir}" "${localDir}/merged"
36+
expect_ret "getmerge" 0 test -s "${localDir}/merged"

‎hadoop-tools/hadoop-compat-bench/shell/cases/copy.t

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/sh
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
. $(dirname "$0")/../misc.sh
19+
20+
echo "Hello World!" > "${localDir}/dat"
21+
22+
echo "1..3"
23+
24+
# 1. copyFromLocal
25+
expect_ret "copyFromLocal" 0 hadoop fs -copyFromLocal "${localDir}/dat" "${baseDir}/"
26+
27+
# 2. cp
28+
hadoop fs -cp "${baseDir}/dat" "${baseDir}/dat2"
29+
expect_ret "cp" 0 hadoop fs -test -f "${baseDir}/dat2"
30+
31+
# 3. copyToLocal
32+
hadoop fs -copyToLocal "${baseDir}/dat2" "${localDir}/"
33+
expect_ret "copyToLocal" 0 test -f "${localDir}/dat2"

‎hadoop-tools/hadoop-compat-bench/shell/cases/directory.t

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/sh
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
. $(dirname "$0")/../misc.sh
19+
20+
echo "Hello World!" > "${localDir}/dat"
21+
hadoop fs -put "${localDir}/dat" "${baseDir}/"
22+
23+
echo "1..8"
24+
25+
# 1. mkdir
26+
expect_ret "mkdir" 0 hadoop fs -mkdir -p "${baseDir}/dir/sub"
27+
28+
# 2. ls
29+
expect_lines "ls" 2 ".*dat.*" ".*dir.*" hadoop fs -ls "${baseDir}"
30+
31+
# 3. lsr
32+
expect_lines "lsr" 3 ".*dat.*" ".*dir.*" ".*sub.*" hadoop fs -lsr "${baseDir}"
33+
34+
# 4. count
35+
expect_out "count" ".*13.*" hadoop fs -count "${baseDir}"
36+
37+
# 5. du
38+
expect_out "du" ".*13.*" hadoop fs -du "${baseDir}"
39+
40+
# 6. dus
41+
expect_out "dus" ".*13.*" hadoop fs -dus "${baseDir}"
42+
43+
# 7. df
44+
expect_ret "df" 0 hadoop fs -df "${baseDir}"
45+
46+
# 8. find
47+
expect_out "find" ".*dat.*" hadoop fs -find "${baseDir}" -name "dat" -print

‎hadoop-tools/hadoop-compat-bench/shell/cases/fileinfo.t

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/sh
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
. $(dirname "$0")/../misc.sh
19+
20+
echo "Hello World!" > "${localDir}/dat"
21+
hadoop fs -put "${localDir}/dat" "${baseDir}/"
22+
23+
echo "1..2"
24+
25+
# 1. stat
26+
expect_out "stat" "size:13" hadoop fs -stat "size:%b" "${baseDir}/dat"
27+
28+
# 2. test
29+
expect_ret "test" 0 hadoop fs -test -f "${baseDir}/dat"

‎hadoop-tools/hadoop-compat-bench/shell/cases/move.t

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/sh
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
. $(dirname "$0")/../misc.sh
19+
20+
echo "Hello World!" > "${localDir}/dat"
21+
22+
echo "1..2"
23+
24+
# 1. moveFromLocal
25+
expect_ret "moveFromLocal" 0 hadoop fs -moveFromLocal "${localDir}/dat" "${baseDir}/"
26+
27+
# 2. mv
28+
hadoop fs -mv "${baseDir}/dat" "${baseDir}/dat2"
29+
expect_ret "mv" 0 hadoop fs -test -f "${baseDir}/dat2"
30+
31+
# moveToLocal is not achieved on HDFS
32+
# hadoop fs -moveToLocal "${baseDir}/dat2" "${localDir}/"
33+
# expect_ret "moveToLocal" 0 test -f "${localDir}/dat2"

0 commit comments

Comments
 (0)