Skip to content

Commit a290e94

Browse files
Jing9Jing9
committed
HDFS-5487. Introduce unit test for TokenAspect. Contributed by Haohui Mai.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1541776 13f79535-47bb-0310-9956-ffa450edef68
1 parent d43207b commit a290e94

File tree

2 files changed

+300
-0
lines changed

2 files changed

+300
-0
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,8 @@ Release 2.3.0 - UNRELEASED
487487
HDFS-5440. Extract the logic of handling delegation tokens in HftpFileSystem
488488
to the TokenAspect class. (Haohui Mai via jing9)
489489

490+
HDFS-5487. Introduce unit test for TokenAspect. (Haohui Mai via jing9)
491+
490492
OPTIMIZATIONS
491493

492494
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java‎

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.hdfs.web;
20+
21+
import static org.junit.Assert.assertEquals;
22+
import static org.junit.Assert.assertNotNull;
23+
import static org.junit.Assert.assertNull;
24+
import static org.mockito.Matchers.anyString;
25+
import static org.mockito.Mockito.doReturn;
26+
import static org.mockito.Mockito.doThrow;
27+
import static org.mockito.Mockito.never;
28+
import static org.mockito.Mockito.spy;
29+
import static org.mockito.Mockito.verify;
30+
31+
import java.io.FileNotFoundException;
32+
import java.io.IOException;
33+
import java.net.URI;
34+
import java.net.URISyntaxException;
35+
36+
import org.apache.hadoop.conf.Configuration;
37+
import org.apache.hadoop.fs.DelegationTokenRenewer;
38+
import org.apache.hadoop.fs.FSDataInputStream;
39+
import org.apache.hadoop.fs.FSDataOutputStream;
40+
import org.apache.hadoop.fs.FileStatus;
41+
import org.apache.hadoop.fs.FileSystem;
42+
import org.apache.hadoop.fs.Path;
43+
import org.apache.hadoop.fs.permission.FsPermission;
44+
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
45+
import org.apache.hadoop.io.Text;
46+
import org.apache.hadoop.security.SecurityUtilTestHelper;
47+
import org.apache.hadoop.security.UserGroupInformation;
48+
import org.apache.hadoop.security.token.Token;
49+
import org.apache.hadoop.security.token.TokenIdentifier;
50+
import org.apache.hadoop.util.Progressable;
51+
import org.junit.Test;
52+
import org.mockito.Mockito;
53+
import org.mockito.internal.util.reflection.Whitebox;
54+
55+
public class TestTokenAspect {
56+
57+
private static class DummyFs extends FileSystem implements
58+
DelegationTokenRenewer.Renewable, TokenAspect.TokenManagementDelegator {
59+
60+
private static final Text TOKEN_KIND = new Text("DummyFS Token");
61+
private boolean emulateSecurityEnabled;
62+
private TokenAspect<DummyFs> tokenAspect;
63+
private UserGroupInformation ugi = UserGroupInformation
64+
.createUserForTesting("foo", new String[] { "bar" });
65+
private URI uri;
66+
67+
@Override
68+
public FSDataOutputStream append(Path f, int bufferSize,
69+
Progressable progress) throws IOException {
70+
return null;
71+
}
72+
73+
@Override
74+
public void cancelDelegationToken(Token<?> token) throws IOException {
75+
}
76+
77+
@Override
78+
public FSDataOutputStream create(Path f, FsPermission permission,
79+
boolean overwrite, int bufferSize, short replication, long blockSize,
80+
Progressable progress) throws IOException {
81+
return null;
82+
}
83+
84+
@Override
85+
public boolean delete(Path f, boolean recursive) throws IOException {
86+
return false;
87+
}
88+
89+
@Override
90+
public URI getCanonicalUri() {
91+
return super.getCanonicalUri();
92+
}
93+
94+
@Override
95+
public FileStatus getFileStatus(Path f) throws IOException {
96+
return null;
97+
}
98+
99+
@Override
100+
public Token<?> getRenewToken() {
101+
return null;
102+
}
103+
104+
@Override
105+
public URI getUri() {
106+
return uri;
107+
}
108+
109+
@Override
110+
public Path getWorkingDirectory() {
111+
return null;
112+
}
113+
114+
@Override
115+
public void initialize(URI name, Configuration conf) throws IOException {
116+
super.initialize(name, conf);
117+
setConf(conf);
118+
try {
119+
this.uri = new URI(name.getScheme(), name.getAuthority(), null, null,
120+
null);
121+
} catch (URISyntaxException e) {
122+
throw new IllegalArgumentException(e);
123+
}
124+
125+
tokenAspect = new TokenAspect<DummyFs>(this, DummyFs.TOKEN_KIND);
126+
if (emulateSecurityEnabled || UserGroupInformation.isSecurityEnabled()) {
127+
tokenAspect.initDelegationToken(ugi);
128+
}
129+
}
130+
131+
@Override
132+
public FileStatus[] listStatus(Path f) throws FileNotFoundException,
133+
IOException {
134+
return null;
135+
}
136+
137+
@Override
138+
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
139+
return false;
140+
}
141+
142+
@Override
143+
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
144+
return null;
145+
}
146+
147+
@Override
148+
public boolean rename(Path src, Path dst) throws IOException {
149+
return false;
150+
}
151+
152+
@Override
153+
public long renewDelegationToken(Token<?> token) throws IOException {
154+
return 0;
155+
}
156+
157+
@Override
158+
public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
159+
}
160+
161+
@Override
162+
public void setWorkingDirectory(Path new_dir) {
163+
}
164+
}
165+
166+
@Test
167+
public void testGetRemoteToken() throws IOException, URISyntaxException {
168+
Configuration conf = new Configuration();
169+
UserGroupInformation.setConfiguration(conf);
170+
DummyFs fs = spy(new DummyFs());
171+
Token<TokenIdentifier> token = new Token<TokenIdentifier>(new byte[0],
172+
new byte[0], DummyFs.TOKEN_KIND, new Text("127.0.0.1:1234"));
173+
174+
doReturn(token).when(fs).getDelegationToken(anyString());
175+
fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf);
176+
177+
fs.tokenAspect.ensureTokenInitialized();
178+
179+
// Select a token, store and renew it
180+
verify(fs).setDelegationToken(token);
181+
assertNotNull(Whitebox.getInternalState(fs.tokenAspect, "dtRenewer"));
182+
assertNotNull(Whitebox.getInternalState(fs.tokenAspect, "action"));
183+
}
184+
185+
@Test
186+
public void testGetRemoteTokenFailure() throws IOException,
187+
URISyntaxException {
188+
Configuration conf = new Configuration();
189+
UserGroupInformation.setConfiguration(conf);
190+
DummyFs fs = spy(new DummyFs());
191+
IOException e = new IOException();
192+
doThrow(e).when(fs).getDelegationToken(anyString());
193+
194+
fs.emulateSecurityEnabled = true;
195+
fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf);
196+
try {
197+
fs.tokenAspect.ensureTokenInitialized();
198+
} catch (IOException exc) {
199+
assertEquals(e, exc);
200+
}
201+
}
202+
203+
@Test
204+
public void testInitWithNoTokens() throws IOException, URISyntaxException {
205+
Configuration conf = new Configuration();
206+
UserGroupInformation.setConfiguration(conf);
207+
DummyFs fs = spy(new DummyFs());
208+
doReturn(null).when(fs).getDelegationToken(anyString());
209+
fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf);
210+
211+
fs.tokenAspect.ensureTokenInitialized();
212+
213+
// No token will be selected.
214+
verify(fs, never()).setDelegationToken(
215+
Mockito.<Token<? extends TokenIdentifier>> any());
216+
}
217+
218+
@Test
219+
public void testInitWithUGIToken() throws IOException, URISyntaxException {
220+
Configuration conf = new Configuration();
221+
UserGroupInformation.setConfiguration(conf);
222+
DummyFs fs = spy(new DummyFs());
223+
doReturn(null).when(fs).getDelegationToken(anyString());
224+
225+
Token<TokenIdentifier> token = new Token<TokenIdentifier>(new byte[0],
226+
new byte[0], DummyFs.TOKEN_KIND, new Text("127.0.0.1:1234"));
227+
fs.ugi.addToken(token);
228+
fs.ugi.addToken(new Token<TokenIdentifier>(new byte[0], new byte[0],
229+
new Text("Other token"), new Text("127.0.0.1:8021")));
230+
assertEquals("wrong tokens in user", 2, fs.ugi.getTokens().size());
231+
232+
fs.emulateSecurityEnabled = true;
233+
fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf);
234+
fs.tokenAspect.ensureTokenInitialized();
235+
236+
// Select a token from ugi (not from the remote host), store it but don't
237+
// renew it
238+
verify(fs).setDelegationToken(token);
239+
verify(fs, never()).getDelegationToken(anyString());
240+
assertNull(Whitebox.getInternalState(fs.tokenAspect, "dtRenewer"));
241+
assertNull(Whitebox.getInternalState(fs.tokenAspect, "action"));
242+
}
243+
244+
@Test
245+
public void testTokenSelectionPreferences() throws IOException,
246+
URISyntaxException {
247+
Configuration conf = new Configuration();
248+
DummyFs fs = spy(new DummyFs());
249+
doReturn(null).when(fs).getDelegationToken(anyString());
250+
fs.initialize(new URI("dummyfs://localhost:1234"), conf);
251+
TokenAspect<DummyFs> aspect = new TokenAspect<DummyFs>(fs,
252+
DummyFs.TOKEN_KIND);
253+
UserGroupInformation ugi = UserGroupInformation.createUserForTesting("foo",
254+
new String[] { "bar" });
255+
UserGroupInformation.setConfiguration(conf);
256+
257+
// use ip-based tokens
258+
SecurityUtilTestHelper.setTokenServiceUseIp(true);
259+
260+
// test fallback to hdfs token
261+
Token<TokenIdentifier> hdfsToken = new Token<TokenIdentifier>(new byte[0],
262+
new byte[0], DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text(
263+
"127.0.0.1:8020"));
264+
ugi.addToken(hdfsToken);
265+
266+
// test fallback to hdfs token
267+
Token<?> token = aspect.selectDelegationToken(ugi);
268+
assertEquals(hdfsToken, token);
269+
270+
// test dummyfs is favored over hdfs
271+
Token<TokenIdentifier> dummyFsToken = new Token<TokenIdentifier>(
272+
new byte[0], new byte[0], DummyFs.TOKEN_KIND,
273+
new Text("127.0.0.1:1234"));
274+
ugi.addToken(dummyFsToken);
275+
token = aspect.selectDelegationToken(ugi);
276+
assertEquals(dummyFsToken, token);
277+
278+
// switch to using host-based tokens, no token should match
279+
SecurityUtilTestHelper.setTokenServiceUseIp(false);
280+
token = aspect.selectDelegationToken(ugi);
281+
assertNull(token);
282+
283+
// test fallback to hdfs token
284+
hdfsToken = new Token<TokenIdentifier>(new byte[0], new byte[0],
285+
DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text(
286+
"localhost:8020"));
287+
ugi.addToken(hdfsToken);
288+
token = aspect.selectDelegationToken(ugi);
289+
assertEquals(hdfsToken, token);
290+
291+
// test dummyfs is favored over hdfs
292+
dummyFsToken = new Token<TokenIdentifier>(new byte[0], new byte[0],
293+
DummyFs.TOKEN_KIND, new Text("localhost:1234"));
294+
ugi.addToken(dummyFsToken);
295+
token = aspect.selectDelegationToken(ugi);
296+
assertEquals(dummyFsToken, token);
297+
}
298+
}

0 commit comments

Comments
 (0)