Skip to content

Commit 8f6a567

Browse files
steveloughransteveloughran
committed
HADOOP-8545. Filesystem Implementation for OpenStack Swift
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1526848 13f79535-47bb-0310-9956-ffa450edef68
1 parent 3fc5a75 commit 8f6a567

File tree

92 files changed

+14394
-6
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

92 files changed

+14394
-6
lines changed

‎hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -348,4 +348,20 @@
348348
<Method name="waitForServiceToStop" />
349349
<Bug code="JLM" />
350350
</Match>
351-
</FindBugsFilter>
351+
352+
<!--
353+
OpenStack Swift FS module -closes streams in a different method
354+
from where they are opened.
355+
-->
356+
<Match>
357+
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
358+
<Method name="uploadFileAttempt"/>
359+
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
360+
</Match>
361+
<Match>
362+
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
363+
<Method name="uploadFilePartAttempt"/>
364+
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
365+
</Match>
366+
367+
</FindBugsFilter>

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Seekable.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
import org.apache.hadoop.classification.InterfaceAudience;
2323
import org.apache.hadoop.classification.InterfaceStability;
2424

25-
/** Stream that permits seeking. */
25+
/**
26+
* Stream that permits seeking.
27+
*/
2628
@InterfaceAudience.Public
2729
@InterfaceStability.Evolving
2830
public interface Seekable {

‎hadoop-common-project/hadoop-common/src/main/resources/core-default.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,11 @@
492492
</description>
493493
</property>
494494

495+
<property>
496+
<name>fs.swift.impl</name>
497+
<value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
498+
<description>The implementation class of the OpenStack Swift Filesystem</description>
499+
</property>
495500

496501
<property>
497502
<name>fs.automatic.close</name>

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -944,14 +944,20 @@ public void testRenameDirectoryToNonExistentParent() throws Exception {
944944
rename(src, dst, false, true, false, Rename.NONE);
945945
Assert.fail("Expected exception was not thrown");
946946
} catch (IOException e) {
947-
Assert.assertTrue(unwrapException(e) instanceof FileNotFoundException);
947+
IOException ioException = unwrapException(e);
948+
if (!(ioException instanceof FileNotFoundException)) {
949+
throw ioException;
950+
}
948951
}
949952

950953
try {
951954
rename(src, dst, false, true, false, Rename.OVERWRITE);
952955
Assert.fail("Expected exception was not thrown");
953956
} catch (IOException e) {
954-
Assert.assertTrue(unwrapException(e) instanceof FileNotFoundException);
957+
IOException ioException = unwrapException(e);
958+
if (!(ioException instanceof FileNotFoundException)) {
959+
throw ioException;
960+
}
955961
}
956962
}
957963

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ public void testOverwrite() throws IOException {
270270

271271
public void testWriteInNonExistentDirectory() throws IOException {
272272
Path path = path("/test/hadoop/file");
273-
assertFalse("Parent doesn't exist", fs.exists(path.getParent()));
273+
assertFalse("Parent exists", fs.exists(path.getParent()));
274274
createFile(path);
275275

276276
assertTrue("Exists", fs.exists(path));
@@ -280,7 +280,7 @@ public void testWriteInNonExistentDirectory() throws IOException {
280280

281281
public void testDeleteNonExistentFile() throws IOException {
282282
Path path = path("/test/hadoop/file");
283-
assertFalse("Doesn't exist", fs.exists(path));
283+
assertFalse("Path exists: " + path, fs.exists(path));
284284
assertFalse("No deletion", fs.delete(path, true));
285285
}
286286

‎hadoop-project/pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,12 @@
305305
<version>${project.version}</version>
306306
</dependency>
307307

308+
<dependency>
309+
<groupId>org.apache.hadoop</groupId>
310+
<artifactId>hadoop-openstack</artifactId>
311+
<version>${project.version}</version>
312+
</dependency>
313+
308314
<dependency>
309315
<groupId>com.google.guava</groupId>
310316
<artifactId>guava</artifactId>
@@ -335,6 +341,11 @@
335341
<artifactId>commons-httpclient</artifactId>
336342
<version>3.1</version>
337343
</dependency>
344+
<dependency>
345+
<groupId>org.apache.httpcomponents</groupId>
346+
<artifactId>httpclient</artifactId>
347+
<version>4.2.5</version>
348+
</dependency>
338349
<dependency>
339350
<groupId>commons-codec</groupId>
340351
<artifactId>commons-codec</artifactId>

‎hadoop-tools/hadoop-openstack/pom.xml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
18+
<modelVersion>4.0.0</modelVersion>
19+
<parent>
20+
<groupId>org.apache.hadoop</groupId>
21+
<artifactId>hadoop-project</artifactId>
22+
<version>3.0.0-SNAPSHOT</version>
23+
<relativePath>../../hadoop-project</relativePath>
24+
</parent>
25+
<artifactId>hadoop-openstack</artifactId>
26+
<version>3.0.0-SNAPSHOT</version>
27+
<name>Apache Hadoop OpenStack support</name>
28+
<description>
29+
This module contains code to support integration with OpenStack.
30+
Currently this consists of a filesystem client to read data from
31+
and write data to an OpenStack Swift object store.
32+
</description>
33+
<packaging>jar</packaging>
34+
35+
<properties>
36+
<file.encoding>UTF-8</file.encoding>
37+
<downloadSources>true</downloadSources>
38+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
39+
</properties>
40+
41+
<profiles>
42+
<profile>
43+
<id>tests-off</id>
44+
<activation>
45+
<file>
46+
<missing>src/test/resources/auth-keys.xml</missing>
47+
</file>
48+
</activation>
49+
<properties>
50+
<maven.test.skip>true</maven.test.skip>
51+
</properties>
52+
</profile>
53+
<profile>
54+
<id>tests-on</id>
55+
<activation>
56+
<file>
57+
<exists>src/test/resources/auth-keys.xml</exists>
58+
</file>
59+
</activation>
60+
<properties>
61+
<maven.test.skip>false</maven.test.skip>
62+
</properties>
63+
</profile>
64+
65+
</profiles>
66+
67+
<build>
68+
<plugins>
69+
70+
<plugin>
71+
<groupId>org.apache.maven.plugins</groupId>
72+
<artifactId>maven-site-plugin</artifactId>
73+
<dependencies>
74+
<dependency>
75+
<groupId>org.apache.maven.doxia</groupId>
76+
<artifactId>doxia-module-markdown</artifactId>
77+
<version>1.3</version>
78+
</dependency>
79+
</dependencies>
80+
<configuration>
81+
<inputEncoding>UTF-8</inputEncoding>
82+
<outputEncoding>UTF-8</outputEncoding>
83+
</configuration>
84+
</plugin>
85+
<plugin>
86+
<groupId>org.apache.maven.plugins</groupId>
87+
<artifactId>maven-project-info-reports-plugin</artifactId>
88+
89+
<configuration>
90+
<dependencyDetailsEnabled>false</dependencyDetailsEnabled>
91+
<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
92+
</configuration>
93+
</plugin>
94+
</plugins>
95+
</build>
96+
97+
<dependencies>
98+
<dependency>
99+
<groupId>org.apache.hadoop</groupId>
100+
<artifactId>hadoop-common</artifactId>
101+
<scope>compile</scope>
102+
</dependency>
103+
104+
<dependency>
105+
<groupId>org.apache.hadoop</groupId>
106+
<artifactId>hadoop-common</artifactId>
107+
<scope>compile</scope>
108+
<type>test-jar</type>
109+
</dependency>
110+
111+
<dependency>
112+
<groupId>org.codehaus.jackson</groupId>
113+
<artifactId>jackson-mapper-asl</artifactId>
114+
<scope>compile</scope>
115+
</dependency>
116+
<dependency>
117+
<groupId>org.codehaus.jackson</groupId>
118+
<artifactId>jackson-core-asl</artifactId>
119+
<scope>compile</scope>
120+
</dependency>
121+
<dependency>
122+
<groupId>org.apache.httpcomponents</groupId>
123+
<artifactId>httpclient</artifactId>
124+
<scope>compile</scope>
125+
</dependency>
126+
127+
<!-- Used for loading test resources and converting a File to byte[] -->
128+
<dependency>
129+
<groupId>commons-io</groupId>
130+
<artifactId>commons-io</artifactId>
131+
<scope>compile</scope>
132+
</dependency>
133+
134+
<!-- Used for mocking dependencies -->
135+
<dependency>
136+
<groupId>org.mockito</groupId>
137+
<artifactId>mockito-all</artifactId>
138+
<version>1.8.5</version>
139+
<scope>test</scope>
140+
</dependency>
141+
142+
<dependency>
143+
<groupId>junit</groupId>
144+
<artifactId>junit</artifactId>
145+
<scope>compile</scope>
146+
</dependency>
147+
<dependency>
148+
<groupId>com.google.guava</groupId>
149+
<artifactId>guava</artifactId>
150+
<scope>test</scope>
151+
</dependency>
152+
</dependencies>
153+
</project>

‎hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.swift.auth;
20+
21+
import org.codehaus.jackson.annotate.JsonProperty;
22+
23+
/**
24+
* Class that represents authentication request to Openstack Keystone.
25+
* Contains basic authentication information.
26+
* THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
27+
* DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS
28+
*/
29+
public class ApiKeyAuthenticationRequest extends AuthenticationRequest {
30+
/**
31+
* Credentials for login
32+
*/
33+
private ApiKeyCredentials apiKeyCredentials;
34+
35+
/**
36+
* API key auth
37+
* @param tenantName tenant
38+
* @param apiKeyCredentials credentials
39+
*/
40+
public ApiKeyAuthenticationRequest(String tenantName, ApiKeyCredentials apiKeyCredentials) {
41+
this.tenantName = tenantName;
42+
this.apiKeyCredentials = apiKeyCredentials;
43+
}
44+
45+
/**
46+
* @return credentials for login into Keystone
47+
*/
48+
@JsonProperty("RAX-KSKEY:apiKeyCredentials")
49+
public ApiKeyCredentials getApiKeyCredentials() {
50+
return apiKeyCredentials;
51+
}
52+
53+
/**
54+
* @param apiKeyCredentials credentials for login into Keystone
55+
*/
56+
public void setApiKeyCredentials(ApiKeyCredentials apiKeyCredentials) {
57+
this.apiKeyCredentials = apiKeyCredentials;
58+
}
59+
60+
@Override
61+
public String toString() {
62+
return "Auth as " +
63+
"tenant '" + tenantName + "' "
64+
+ apiKeyCredentials;
65+
}
66+
}

0 commit comments

Comments
 (0)