Skip to content

Commit 44af3a7

Browse files
Ethan-ZhangEthan-Zhang
committed
增加了集成测试，修复了几个bug
1. base_path传根目录导致的异常 2. access id key 改为security settings
1 parent 57caca1 commit 44af3a7

File tree

10 files changed

+208
-37
lines changed

10 files changed

+208
-37
lines changed

‎build.gradle

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
import org.elasticsearch.gradle.test.NodeInfo
2-
import org.gradle.api.internal.project.DefaultAntBuilder
3-
41
buildscript {
52
ext {
63
pluginName = project.properties['pluginName']
@@ -45,7 +42,7 @@ noticeFile = rootProject.file('NOTICE.txt')
4542

4643
esplugin {
4744
name pluginName
48-
description 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.'
45+
description 'The Cos Repository plugin adds support for using Tencent Cloud Cos as a repository fro Snapshot/Restore.'
4946
classname 'org.elasticsearch.repositories.cos.COSRepositoryPlugin'
5047
version pluginVersion
5148
licenseFile rootProject.file('LICENSE.txt')
@@ -71,6 +68,34 @@ dependencies {
7168
compile "org.apache.httpcomponents:httpcore:4.4.11"
7269
}
7370

71+
bundlePlugin {
72+
73+
}
74+
75+
test {
76+
exclude '**/CosRepositoryThirdPartyTests.class'
77+
}
78+
79+
String cosAccessKeyId = System.getenv("qcloud_cos_access_key_id")
80+
String cosAccessKeySecret = System.getenv("qcloud_cos_access_key_secret")
81+
String cosBucket = System.getenv("qcloud_cos_bucket")
82+
String cosBasePath = System.getenv("qcloud_cos_base_path")
83+
String cosRegion = System.getenv("qcloud_cos_region")
84+
85+
task thirdPartyTest(type: Test) {
86+
include '**/CosRepositoryThirdPartyTests.class'
87+
systemProperty 'es.allow_insecure_settings', 'true'
88+
systemProperty 'access_key_id', cosAccessKeyId
89+
systemProperty 'access_key_secret', cosAccessKeySecret
90+
systemProperty 'bucket', cosBucket
91+
systemProperty 'base_path', cosBasePath
92+
systemProperty 'region', cosRegion
93+
}
94+
95+
thirdPartyTest {
96+
dependsOn tasks.bundlePlugin
97+
}
98+
7499
// elasticsearch.esplugin task configure
75100
checkstyleMain.enabled = false
76101
checkstyleTest.enabled = false
@@ -80,4 +105,4 @@ thirdPartyAudit.enabled = false
80105
licenseHeaders.enabled = false
81106
forbiddenApisMain.enabled = false
82107
forbiddenApisTest.enabled = false
83-
integTest.enabled = false
108+
licenseTest.enabled = false

‎src/main/java/org/elasticsearch/repositories/cos/COSBlobContainer.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,11 @@ void doMultipartUpload(String blobName, InputStream inputStream, long blobSize)
159159

160160
@Override
161161
public void deleteBlob(String blobName) throws IOException {
162+
deleteBlobIgnoringIfNotExists(blobName);
163+
}
164+
165+
@Override
166+
public void deleteBlobIgnoringIfNotExists(String blobName) throws IOException {
162167
try {
163168
SocketAccess.doPrivilegedVoid(() ->
164169
blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName)));
@@ -187,7 +192,7 @@ public DeleteResult delete() throws IOException {
187192
final List<String> blobsToDelete = new ArrayList<>();
188193
list.getObjectSummaries().forEach(cosObjectSummary -> {
189194
deletedBlobs.incrementAndGet();
190-
deletedBytes.incrementAndGet();
195+
deletedBytes.addAndGet(cosObjectSummary.getSize());
191196
blobsToDelete.add(cosObjectSummary.getKey());
192197
});
193198
if (list.isTruncated()) {
@@ -288,6 +293,13 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
288293
@Override
289294
public Map<String, BlobContainer> children() throws IOException {
290295
try {
296+
Map<String, BlobContainer> a = executeListing(generateListObjectsRequest(keyPath)).stream()
297+
.flatMap(listing -> listing.getCommonPrefixes().stream())
298+
.map(prefix -> prefix.substring(keyPath.length()))
299+
.filter(name -> name.isEmpty() == false)
300+
.map(name -> name.substring(0, name.length() - 1))
301+
.collect(Collectors.toMap(Function.identity(), name -> blobStore.blobContainer(path().add(name))));
302+
291303
return executeListing(generateListObjectsRequest(keyPath)).stream()
292304
.flatMap(listing -> listing.getCommonPrefixes().stream())
293305
.map(prefix -> prefix.substring(keyPath.length()))

‎src/main/java/org/elasticsearch/repositories/cos/COSClientSettings.java

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.elasticsearch.repositories.cos;
22

3+
import org.elasticsearch.common.settings.SecureSetting;
4+
import org.elasticsearch.common.settings.SecureString;
35
import org.elasticsearch.common.settings.Setting;
46
import org.elasticsearch.common.unit.ByteSizeUnit;
57
import org.elasticsearch.common.unit.ByteSizeValue;
@@ -12,16 +14,16 @@ public class COSClientSettings {
1214

1315
public static final Setting<String> REGION =
1416
Setting.simpleString("region", Property.NodeScope, Property.Dynamic);
15-
public static final Setting<String> ACCESS_KEY_ID =
16-
Setting.simpleString("access_key_id", Setting.Property.NodeScope, Setting.Property.Dynamic);
17-
public static final Setting<String> ACCESS_KEY_SECRET = Setting
18-
.simpleString("access_key_secret", Setting.Property.NodeScope, Setting.Property.Dynamic);
17+
public static final Setting<SecureString> ACCESS_KEY_ID =
18+
SecureSetting.secureString("access_key_id", null);
19+
public static final Setting<SecureString> ACCESS_KEY_SECRET =
20+
SecureSetting.secureString("access_key_secret", null);
1921
public static final Setting<String> APP_ID = Setting
2022
.simpleString("app_id", "", Setting.Property.NodeScope, Setting.Property.Dynamic);
2123
public static final Setting<String> BUCKET =
2224
simpleString("bucket", Setting.Property.NodeScope, Setting.Property.Dynamic);
2325
public static final Setting<String> BASE_PATH =
24-
simpleString("base_path", Setting.Property.NodeScope, Setting.Property.Dynamic);
26+
simpleString("base_path", "", Setting.Property.NodeScope, Setting.Property.Dynamic);
2527
public static final Setting<Boolean> COMPRESS =
2628
boolSetting("compress", false, Setting.Property.NodeScope, Setting.Property.Dynamic);
2729
public static final Setting<ByteSizeValue> CHUNK_SIZE =

‎src/main/java/org/elasticsearch/repositories/cos/COSRepository.java

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public class COSRepository extends BlobStoreRepository {
1717
private static final Logger logger = LogManager.getLogger(COSRepository.class);
1818
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
1919
public static final String TYPE = "cos";
20+
2021
private final BlobPath basePath;
2122
private final boolean compress;
2223
private final ByteSizeValue chunkSize;
@@ -36,8 +37,11 @@ public class COSRepository extends BlobStoreRepository {
3637
ThreadPool threadpool) {
3738
super(metadata, COMPRESS_SETTING.get(metadata.settings()), namedXContentRegistry, threadpool);
3839
this.service = cos;
39-
String bucket = getSetting(COSClientSettings.BUCKET, metadata);
40-
String basePath = getSetting(COSClientSettings.BASE_PATH, metadata);
40+
String bucket = COSClientSettings.BUCKET.get(metadata.settings());
41+
if (bucket == null || !Strings.hasLength(bucket)) {
42+
throw new RepositoryException(metadata.name(), "No bucket defined for cos repository");
43+
}
44+
String basePath = COSClientSettings.BASE_PATH.get(metadata.settings());
4145
String app_id = COSClientSettings.APP_ID.get(metadata.settings());
4246
// qcloud-sdk-v5 app_id directly joined with bucket name
4347
if (Strings.hasLength(app_id)) {
@@ -47,17 +51,18 @@ public class COSRepository extends BlobStoreRepository {
4751
this.bucket = bucket;
4852
}
4953

54+
if (basePath.startsWith("/")) {
55+
basePath = basePath.substring(1);
56+
deprecationLogger.deprecated("cos repository base_path trimming the leading `/`, and leading `/` will not be supported for the cos repository in future releases");
57+
}
58+
5059
if (Strings.hasLength(basePath)) {
51-
if (basePath.startsWith("/")) {
52-
basePath = basePath.substring(1);
53-
deprecationLogger.deprecated("cos repository base_path trimming the leading `/`, and leading `/` will not be supported for the cos repository in future releases");
54-
}
5560
this.basePath = new BlobPath().add(basePath);
5661
} else {
5762
this.basePath = BlobPath.cleanPath();
5863
}
59-
this.compress = getSetting(COSClientSettings.COMPRESS, metadata);
60-
this.chunkSize = getSetting(COSClientSettings.CHUNK_SIZE, metadata);
64+
this.compress = COSClientSettings.COMPRESS.get(metadata.settings());
65+
this.chunkSize = COSClientSettings.CHUNK_SIZE.get(metadata.settings());
6166

6267
logger.trace("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}]", bucket,
6368
basePath, chunkSize, compress);
@@ -78,17 +83,4 @@ public BlobPath basePath() {
7883
protected ByteSizeValue chunkSize() {
7984
return chunkSize;
8085
}
81-
82-
public static <T> T getSetting(Setting<T> setting, RepositoryMetaData metadata) {
83-
T value = setting.get(metadata.settings());
84-
if (value == null) {
85-
throw new RepositoryException(metadata.name(),
86-
"Setting [" + setting.getKey() + "] is not defined for repository");
87-
}
88-
if ((value instanceof String) && (Strings.hasText((String) value)) == false) {
89-
throw new RepositoryException(metadata.name(),
90-
"Setting [" + setting.getKey() + "] is empty for repository");
91-
}
92-
return value;
93-
}
9486
}

‎src/main/java/org/elasticsearch/repositories/cos/COSRepositoryPlugin.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,4 @@ public List<Setting<?>> getSettings() {
3737
COSClientSettings.APP_ID, COSClientSettings.BUCKET,
3838
COSClientSettings.BASE_PATH, COSClientSettings.COMPRESS, COSClientSettings.CHUNK_SIZE, COSClientSettings.CosEndPoint);
3939
}
40-
}
40+
}

‎src/main/java/org/elasticsearch/repositories/cos/COSService.java

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,13 @@
99
import org.elasticsearch.common.Strings;
1010
import org.elasticsearch.common.unit.ByteSizeUnit;
1111
import org.elasticsearch.common.unit.ByteSizeValue;
12+
import org.elasticsearch.repositories.RepositoryException;
13+
14+
import java.io.Closeable;
15+
import java.io.IOException;
1216

1317
//TODO: 考虑是否需要继承closeable，处理连接池等问题
14-
public class COSService {
18+
public class COSService implements Closeable {
1519

1620
private COSClient client;
1721
public static final ByteSizeValue MAX_SINGLE_FILE_SIZE = new ByteSizeValue(5, ByteSizeUnit.GB);
@@ -21,9 +25,12 @@ public class COSService {
2125
}
2226

2327
private synchronized COSClient createClient(RepositoryMetaData metaData) {
24-
String access_key_id = COSRepository.getSetting(COSClientSettings.ACCESS_KEY_ID, metaData);
25-
String access_key_secret = COSRepository.getSetting(COSClientSettings.ACCESS_KEY_SECRET, metaData);
26-
String region = COSRepository.getSetting(COSClientSettings.REGION, metaData);
28+
String access_key_id = COSClientSettings.ACCESS_KEY_ID.get(metaData.settings()).toString();
29+
String access_key_secret = COSClientSettings.ACCESS_KEY_SECRET.get(metaData.settings()).toString();
30+
String region = COSClientSettings.REGION.get(metaData.settings());
31+
if (region == null || !Strings.hasLength(region)) {
32+
throw new RepositoryException(metaData.name(), "No region defined for cos repository");
33+
}
2734
String endPoint = COSClientSettings.CosEndPoint.get(metaData.settings());
2835

2936
COSCredentials cred = new BasicCOSCredentials(access_key_id, access_key_secret);
@@ -40,4 +47,9 @@ public COSClient getClient() {
4047
return this.client;
4148
}
4249

50+
@Override
51+
public void close() throws IOException {
52+
this.client.shutdown();
53+
}
54+
4355
}

‎src/main/plugin-metadata/plugin-security.policy

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,7 @@ grant {
2323

2424
// cos client opens socket connections for to access repository
2525
permission java.net.SocketPermission "*", "connect";
26+
27+
// only for tests : org.elasticsearch.repositories.s3.S3RepositoryPlugin
28+
permission java.util.PropertyPermission "es.allow_insecure_settings", "read,write";
2629
};

‎src/test/java/org/elasticsearch/repositories/cos/CosClientSettingsTests.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package org.elasticsearch.repositories.cos;
2+
3+
import org.elasticsearch.test.ESTestCase;
4+
5+
public class CosClientSettingsTests extends ESTestCase {
6+
public void testDefaultClientSettingsCanBeSet() {
7+
System.out.println("adsfasfsa");
8+
assertEquals(1, 1);
9+
}
10+
}

‎src/test/java/org/elasticsearch/repositories/cos/CosRepositoryThirdPartyTests.java

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package org.elasticsearch.repositories.cos;
2+
3+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
4+
import org.elasticsearch.client.Client;
5+
import org.elasticsearch.common.blobstore.BlobMetaData;
6+
import org.elasticsearch.common.blobstore.BlobPath;
7+
import org.elasticsearch.common.settings.MockSecureSettings;
8+
import org.elasticsearch.common.settings.SecureSettings;
9+
import org.elasticsearch.common.settings.Settings;
10+
import org.elasticsearch.plugins.Plugin;
11+
import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
12+
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
13+
14+
import java.util.Collection;
15+
import java.util.Map;
16+
import java.util.concurrent.Executor;
17+
import java.util.concurrent.TimeUnit;
18+
19+
import static org.hamcrest.Matchers.blankOrNullString;
20+
import static org.hamcrest.Matchers.equalTo;
21+
import static org.hamcrest.core.IsNot.not;
22+
23+
public class CosRepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
24+
25+
@Override
26+
protected Collection<Class<? extends Plugin>> getPlugins() {
27+
return pluginList(COSRepositoryPlugin.class);
28+
}
29+
30+
protected SecureSettings credentials() {
31+
assertThat(System.getProperty("access_key_id"), not(blankOrNullString()));
32+
assertThat(System.getProperty("access_key_secret"), not(blankOrNullString()));
33+
assertThat(System.getProperty("bucket"), not(blankOrNullString()));
34+
35+
MockSecureSettings secureSettings = new MockSecureSettings();
36+
secureSettings.setString("access_key_id", System.getProperty("access_key_id"));
37+
secureSettings.setString("access_key_secret", System.getProperty("access_key_secret"));
38+
return secureSettings;
39+
}
40+
41+
@Override
42+
protected void createRepository(String repoName) {
43+
final Client client = client();
44+
45+
MockSecureSettings secureSettings = new MockSecureSettings();
46+
secureSettings.setString("access_key_id", System.getProperty("access_key_id"));
47+
secureSettings.setString("access_key_secret", System.getProperty("access_key_secret"));
48+
49+
AcknowledgedResponse putReposirotyResponse =
50+
client.admin().cluster().preparePutRepository(repoName)
51+
.setType(COSRepository.TYPE)
52+
.setSettings(Settings.builder()
53+
.setSecureSettings(secureSettings)
54+
.put("access_key_id", System.getProperty("access_key_id"))
55+
.put("access_key_secret", System.getProperty("access_key_secret"))
56+
.put("bucket",System.getProperty("bucket"))
57+
.put("base_path",System.getProperty("base_path"))
58+
.put("region",System.getProperty("region")))
59+
.get();
60+
61+
assertThat(putReposirotyResponse.isAcknowledged(), equalTo(true));
62+
}
63+
64+
@Override
65+
protected boolean assertCorruptionVisible(BlobStoreRepository repo, Executor genericExec) throws Exception {
66+
// S3 is only eventually consistent for the list operations used by this assertions so we retry for 10 minutes assuming that
67+
// listing operations will become consistent within these 10 minutes.
68+
assertBusy(() -> assertTrue(super.assertCorruptionVisible(repo, genericExec)), 30L, TimeUnit.SECONDS);
69+
return true;
70+
}
71+
72+
@Override
73+
protected void assertConsistentRepository(BlobStoreRepository repo, Executor executor) throws Exception {
74+
// S3 is only eventually consistent for the list operations used by this assertions so we retry for 10 minutes assuming that
75+
// listing operations will become consistent within these 10 minutes.
76+
assertBusy(() -> super.assertConsistentRepository(repo, executor), 30L, TimeUnit.SECONDS);
77+
}
78+
79+
protected void assertBlobsByPrefix(BlobPath path, String prefix, Map<String, BlobMetaData> blobs) throws Exception {
80+
// AWS S3 is eventually consistent so we retry for 10 minutes assuming a list operation will never take longer than that
81+
// to become consistent.
82+
assertBusy(() -> super.assertBlobsByPrefix(path, prefix, blobs), 30L, TimeUnit.SECONDS);
83+
}
84+
85+
@Override
86+
protected void assertChildren(BlobPath path, Collection<String> children) throws Exception {
87+
// AWS S3 is eventually consistent so we retry for 10 minutes assuming a list operation will never take longer than that
88+
// to become consistent.
89+
assertBusy(() -> super.assertChildren(path, children), 30L, TimeUnit.SECONDS);
90+
}
91+
92+
@Override
93+
protected void assertDeleted(BlobPath path, String name) throws Exception {
94+
// AWS S3 is eventually consistent so we retry for 10 minutes assuming a list operation will never take longer than that
95+
// to become consistent.
96+
assertBusy(() -> super.assertDeleted(path, name), 30L, TimeUnit.SECONDS);
97+
}
98+
99+
}

‎src/test/resources/rest-api-spec.test.repository_cos/10_basic.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Integration tests for repository-cos
2+
#
3+
"Plugin repository-cos is loaded":
4+
- skip:
5+
reason: "contains is a newly added assertion"
6+
features: contains
7+
- do:
8+
cluster.state: {}
9+
10+
# Get master node id
11+
- set: { master_node: master }
12+
13+
- do:
14+
nodes.info: {}
15+
16+
- contains: { nodes.$master.plugins: { name: repository-cos } }

0 commit comments

Comments
 (0)