Skip to content

Commit 4dc7313

Browse files
wojiaodoubaolijinglun
wojiaodoubao
authored and
lijinglun
committed
HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation.
Contributed by: ZhengHu, SunXin, XianyinXin, Rascal Wu, FangBo, Yuanzhihuan.
1 parent 4baa167 commit 4dc7313

File tree

112 files changed

+14203
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

112 files changed

+14203
-0
lines changed

‎hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -125,5 +125,10 @@
125125
<artifactId>hadoop-huaweicloud</artifactId>
126126
<scope>compile</scope>
127127
</dependency>
128+
<dependency>
129+
<groupId>org.apache.hadoop</groupId>
130+
<artifactId>hadoop-tos</artifactId>
131+
<scope>compile</scope>
132+
</dependency>
128133
</dependencies>
129134
</project>

‎hadoop-cloud-storage-project/hadoop-tos/dev-support/findbugs-exclude.xml

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<FindBugsFilter>
18+
<Match>
19+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectOutputStream"/>
20+
<or>
21+
<Bug pattern="IS2_INCONSISTENT_SYNC" />
22+
<Bug pattern="NP_NULL_ON_SOME_PATH" />
23+
</or>
24+
</Match>
25+
<Match>
26+
<Class name="org.apache.hadoop.fs.tosfs.conf.TosKeys"/>
27+
<Bug pattern="MS_MUTABLE_ARRAY" />
28+
</Match>
29+
<Match>
30+
<Class name="org.apache.hadoop.fs.tosfs.object.Constants"/>
31+
<Bug pattern="MS_MUTABLE_ARRAY" />
32+
</Match>
33+
<Match>
34+
<Class name="org.apache.hadoop.fs.tosfs.RawFileStatus"/>
35+
<or>
36+
<Bug pattern="EI_EXPOSE_REP" />
37+
<Bug pattern="EI_EXPOSE_REP2" />
38+
</or>
39+
</Match>
40+
<Match>
41+
<Class name="org.apache.hadoop.fs.tosfs.TosChecksum"/>
42+
<or>
43+
<Bug pattern="EI_EXPOSE_REP" />
44+
<Bug pattern="EI_EXPOSE_REP2" />
45+
</or>
46+
</Match>
47+
<Match>
48+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectContent"/>
49+
<or>
50+
<Bug pattern="EI_EXPOSE_REP" />
51+
<Bug pattern="EI_EXPOSE_REP2" />
52+
</or>
53+
</Match>
54+
<Match>
55+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectInfo"/>
56+
<or>
57+
<Bug pattern="EI_EXPOSE_REP" />
58+
<Bug pattern="EI_EXPOSE_REP2" />
59+
</or>
60+
</Match>
61+
<Match>
62+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectMultiRangeInputStream"/>
63+
<or>
64+
<Bug pattern="EI_EXPOSE_REP" />
65+
<Bug pattern="EI_EXPOSE_REP2" />
66+
</or>
67+
</Match>
68+
<Match>
69+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectRangeInputStream"/>
70+
<or>
71+
<Bug pattern="EI_EXPOSE_REP" />
72+
<Bug pattern="EI_EXPOSE_REP2" />
73+
</or>
74+
</Match>
75+
<Match>
76+
<Class name="org.apache.hadoop.fs.tosfs.object.tos.GetObjectOutput"/>
77+
<or>
78+
<Bug pattern="EI_EXPOSE_REP" />
79+
<Bug pattern="EI_EXPOSE_REP2" />
80+
</or>
81+
</Match>
82+
<Match>
83+
<Class name="org.apache.hadoop.fs.tosfs.object.tos.ChainTOSInputStream"/>
84+
<Bug pattern="EI_EXPOSE_REP" />
85+
</Match>
86+
<Match>
87+
<Bug pattern="ICAST_QUESTIONABLE_UNSIGNED_RIGHT_SHIFT" />
88+
</Match>
89+
</FindBugsFilter>

‎hadoop-cloud-storage-project/hadoop-tos/pom.xml

+183
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
18+
<modelVersion>4.0.0</modelVersion>
19+
<parent>
20+
<groupId>org.apache.hadoop</groupId>
21+
<artifactId>hadoop-project</artifactId>
22+
<version>3.5.0-SNAPSHOT</version>
23+
<relativePath>../../hadoop-project</relativePath>
24+
</parent>
25+
26+
<artifactId>hadoop-tos</artifactId>
27+
<version>3.5.0-SNAPSHOT</version>
28+
<name>Apache Hadoop Volcano Engine Services support</name>
29+
<description>
30+
This module contains code to support integration with Volcano Engine TOS.
31+
It also declares the dependencies needed to work with Volcano Engine services.
32+
</description>
33+
<packaging>jar</packaging>
34+
35+
<properties>
36+
<file.encoding>UTF-8</file.encoding>
37+
<ve-tos-java-sdk.version>2.8.9</ve-tos-java-sdk.version>
38+
</properties>
39+
40+
<dependencies>
41+
<dependency>
42+
<groupId>org.apache.hadoop</groupId>
43+
<artifactId>hadoop-common</artifactId>
44+
<scope>provided</scope>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.apache.hadoop</groupId>
48+
<artifactId>hadoop-mapreduce-client-core</artifactId>
49+
<scope>provided</scope>
50+
</dependency>
51+
<dependency>
52+
<groupId>com.volcengine</groupId>
53+
<artifactId>ve-tos-java-sdk-hadoop</artifactId>
54+
<version>${ve-tos-java-sdk.version}</version>
55+
<exclusions>
56+
<exclusion>
57+
<groupId>org.slf4j</groupId>
58+
<artifactId>slf4j-api</artifactId>
59+
</exclusion>
60+
<exclusion>
61+
<groupId>org.jetbrains</groupId>
62+
<artifactId>annotations</artifactId>
63+
</exclusion>
64+
<exclusion>
65+
<groupId>com.fasterxml.jackson.core</groupId>
66+
<artifactId>jackson-annotations</artifactId>
67+
</exclusion>
68+
<exclusion>
69+
<groupId>com.fasterxml.jackson.core</groupId>
70+
<artifactId>jackson-databind</artifactId>
71+
</exclusion>
72+
<exclusion>
73+
<groupId>com.fasterxml.jackson.core</groupId>
74+
<artifactId>jackson-core</artifactId>
75+
</exclusion>
76+
</exclusions>
77+
78+
</dependency>
79+
80+
<!-- Test dependencies -->
81+
<dependency>
82+
<groupId>org.apache.hadoop</groupId>
83+
<artifactId>hadoop-common</artifactId>
84+
<scope>test</scope>
85+
<type>test-jar</type>
86+
</dependency>
87+
<!-- Artifacts needed to bring up a Mini MR Yarn cluster-->
88+
<dependency>
89+
<groupId>org.apache.hadoop</groupId>
90+
<artifactId>hadoop-mapreduce-examples</artifactId>
91+
<scope>test</scope>
92+
</dependency>
93+
<dependency>
94+
<groupId>org.apache.hadoop</groupId>
95+
<artifactId>hadoop-minicluster</artifactId>
96+
<scope>test</scope>
97+
</dependency>
98+
<dependency>
99+
<groupId>org.apache.hadoop</groupId>
100+
<artifactId>hadoop-distcp</artifactId>
101+
<version>${hadoop.version}</version>
102+
<scope>test</scope>
103+
</dependency>
104+
<dependency>
105+
<groupId>org.apache.hadoop</groupId>
106+
<artifactId>hadoop-distcp</artifactId>
107+
<version>${hadoop.version}</version>
108+
<scope>test</scope>
109+
<type>test-jar</type>
110+
</dependency>
111+
112+
<dependency>
113+
<groupId>org.assertj</groupId>
114+
<artifactId>assertj-core</artifactId>
115+
<scope>test</scope>
116+
</dependency>
117+
<dependency>
118+
<groupId>junit</groupId>
119+
<artifactId>junit</artifactId>
120+
<scope>test</scope>
121+
</dependency>
122+
<dependency>
123+
<groupId>org.mockito</groupId>
124+
<artifactId>mockito-core</artifactId>
125+
<version>4.11.0</version>
126+
<scope>test</scope>
127+
</dependency>
128+
</dependencies>
129+
130+
<build>
131+
<plugins>
132+
<plugin>
133+
<groupId>org.apache.maven.plugins</groupId>
134+
<artifactId>maven-surefire-plugin</artifactId>
135+
<configuration>
136+
<parallel>classes</parallel>
137+
<threadCount>1</threadCount>
138+
<perCoreThreadCount>true</perCoreThreadCount>
139+
<rerunFailingTestsCount>2</rerunFailingTestsCount>
140+
<forkCount>8</forkCount>
141+
<argLine>-Xmx2048m</argLine>
142+
</configuration>
143+
</plugin>
144+
145+
<plugin>
146+
<groupId>org.apache.maven.plugins</groupId>
147+
<artifactId>maven-shade-plugin</artifactId>
148+
<executions>
149+
<execution>
150+
<phase>package</phase>
151+
<goals>
152+
<goal>shade</goal>
153+
</goals>
154+
<configuration>
155+
<createDependencyReducedPom>true</createDependencyReducedPom>
156+
<filters>
157+
<filter>
158+
<artifact>*:*</artifact>
159+
<excludes>
160+
<exclude>META-INF/*.SF</exclude>
161+
<exclude>META-INF/*.DSA</exclude>
162+
<exclude>META-INF/*.RSA</exclude>
163+
</excludes>
164+
</filter>
165+
</filters>
166+
</configuration>
167+
</execution>
168+
</executions>
169+
</plugin>
170+
171+
<plugin>
172+
<groupId>com.github.spotbugs</groupId>
173+
<artifactId>spotbugs-maven-plugin</artifactId>
174+
<configuration>
175+
<xmlOutput>true</xmlOutput>
176+
<excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
177+
</excludeFilterFile>
178+
<effort>Max</effort>
179+
</configuration>
180+
</plugin>
181+
</plugins>
182+
</build>
183+
</project>

‎hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFS.java

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.tosfs;
20+
21+
import org.apache.hadoop.conf.Configuration;
22+
import org.apache.hadoop.fs.DelegateToFileSystem;
23+
24+
import java.io.IOException;
25+
import java.net.URI;
26+
import java.net.URISyntaxException;
27+
28+
/**
29+
* The implementation class of the raw AbstractFileSystem. If you want to use object storage as
30+
* YARN’s resource storage dir via the fs.defaultFS configuration property in Hadoop’s
31+
* core-site.xml, you should add this configuration to Hadoop's core-site.xml.
32+
* <pre>
33+
* fs.AbstractFileSystem.{scheme}.impl=io.proton.fs.RawFS.
34+
* </pre>
35+
*/
36+
public class RawFS extends DelegateToFileSystem {
37+
private static final int TOS_DEFAULT_PORT = -1;
38+
39+
public RawFS(URI uri, Configuration conf) throws IOException, URISyntaxException {
40+
super(uri, new RawFileSystem(), conf, uri.getScheme(), false);
41+
}
42+
43+
@Override
44+
public int getUriDefaultPort() {
45+
return TOS_DEFAULT_PORT;
46+
}
47+
}

0 commit comments

Comments
 (0)