Skip to content

Commit 888028a

Browse files
huiguangjunhuiguangjun
committed
before re-uploading, check the upload id is valid.
1 parent 8499acc commit 888028a

File tree

5 files changed

+143
-0
lines changed

5 files changed

+143
-0
lines changed

‎sdk/src/resumable/ResumableCopier.cc‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,26 @@ int ResumableCopier::prepare(OssError& err)
238238
return 0;
239239
}
240240

241+
bool ResumableCopier::doesNotExistUploadId() {
242+
// The upload ID may be invalid, or the upload may have been aborted or completed.
243+
auto listRequest = ListPartsRequest(request_.Bucket(), request_.Key(), record_.uploadID);
244+
if (!request_.EncodingType().empty()) {
245+
listRequest.setEncodingType(request_.EncodingType());
246+
}
247+
if (request_.RequestPayer() == RequestPayer::Requester) {
248+
listRequest.setRequestPayer(request_.RequestPayer());
249+
}
250+
listRequest.setMaxParts(1);
251+
auto listOutcome = this->client_->ListParts(listRequest);
252+
253+
if (!listOutcome.isSuccess() &&
254+
listOutcome.error().Code() == "NoSuchUpload") {
255+
return true;
256+
}
257+
258+
return false;
259+
}
260+
241261
int ResumableCopier::validateRecord()
242262
{
243263
auto record = record_;
@@ -264,6 +284,11 @@ int ResumableCopier::validateRecord()
264284
if (md5Sum != record.md5Sum) {
265285
return ARG_ERROR_COPY_RECORD_INVALID;
266286
}
287+
288+
if (doesNotExistUploadId()) {
289+
return ARG_ERROR_COPY_RECORD_INVALID;
290+
}
291+
267292
return 0;
268293
}
269294

‎sdk/src/resumable/ResumableCopier.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ namespace OSS
5050
int prepare(OssError& err);
5151
void initRecord(const std::string &uploadID);
5252
int getPartsToUploadCopy(OssError &err, PartList &partsCopied, PartList &partsToUpload);
53+
bool doesNotExistUploadId();
5354

5455
const MultiCopyObjectRequest request_;
5556
MultiCopyRecord record_;

‎sdk/src/resumable/ResumableUploader.cc‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,26 @@ int ResumableUploader::prepare(OssError& err)
255255
return 0;
256256
}
257257

258+
bool ResumableUploader::doesNotExistUploadId() {
259+
// The upload ID may be invalid, or the upload may have been aborted or completed.
260+
auto listRequest = ListPartsRequest(request_.Bucket(), request_.Key(), record_.uploadID);
261+
if (!request_.EncodingType().empty()) {
262+
listRequest.setEncodingType(request_.EncodingType());
263+
}
264+
if (request_.RequestPayer() == RequestPayer::Requester) {
265+
listRequest.setRequestPayer(request_.RequestPayer());
266+
}
267+
listRequest.setMaxParts(1);
268+
auto listOutcome = this->client_->ListParts(listRequest);
269+
270+
if (!listOutcome.isSuccess() &&
271+
listOutcome.error().Code() == "NoSuchUpload") {
272+
return true;
273+
}
274+
275+
return false;
276+
}
277+
258278
int ResumableUploader::validateRecord()
259279
{
260280
if (record_.size != objectSize_ || record_.mtime != request_.ObjectMtime()){
@@ -272,6 +292,11 @@ int ResumableUploader::validateRecord()
272292
if (md5Sum != record_.md5Sum){
273293
return ARG_ERROR_UPLOAD_RECORD_INVALID;
274294
}
295+
296+
if (doesNotExistUploadId()) {
297+
return ARG_ERROR_UPLOAD_RECORD_INVALID;
298+
}
299+
275300
return 0;
276301
}
277302

‎sdk/src/resumable/ResumableUploader.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ namespace OSS
5757
virtual void genRecordPath();
5858
virtual int loadRecord();
5959
virtual int prepare(OssError& err);
60+
bool doesNotExistUploadId();
6061

6162
const UploadObjectRequest& request_;
6263
UploadRecord record_;

‎test/src/MultipartUpload/ResumableObjectTest.cc‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,58 @@ class ResumableObjectTest : public::testing::Test {
655655
EXPECT_EQ(RemoveDirectory(checkpointKey), true);
656656
}
657657

658+
TEST_F(ResumableObjectTest, NormalResumableUploadRetryWithUploadIdAbortTest)
659+
{
660+
std::string key = TestUtils::GetObjectKey("NormalResumableUploadRetryWithUploadIdAbortTest");
661+
std::string tmpFile = TestUtils::GetTargetFileName("NormalResumableUploadRetryWithUploadIdAbortTest").append(".tmp");
662+
int num = 2 + rand() % 10;
663+
TestUtils::WriteRandomDatatoFile(tmpFile, 1024 * 100 * num);
664+
std::string sourceFileMd5 = TestUtils::GetFileMd5(tmpFile);
665+
std::string checkpointKey = TestUtils::GetObjectKey("checkpoint");
666+
EXPECT_EQ(CreateDirectory(checkpointKey), true);
667+
EXPECT_EQ(IsDirectoryExist(checkpointKey), true);
668+
669+
// resumable upload object failed
670+
UploadObjectRequest request(BucketName, key, tmpFile);
671+
request.setThreadNum(1);
672+
request.setPartSize(102400);
673+
request.setFlags(request.Flags() | UploadPartFailedFlag);
674+
request.setCheckpointDir(checkpointKey);
675+
auto outcome = Client->ResumableUploadObject(request);
676+
EXPECT_EQ(outcome.isSuccess(), false);
677+
678+
// abort upload Id
679+
ListMultipartUploadsRequest lmuRequest(BucketName);
680+
lmuRequest.setPrefix(key);
681+
auto lmuOutcome = Client->ListMultipartUploads(lmuRequest);
682+
EXPECT_EQ(lmuOutcome.isSuccess(), true);
683+
EXPECT_EQ(lmuOutcome.result().MultipartUploadList().size(), 1L);
684+
auto uploadId = lmuOutcome.result().MultipartUploadList()[0].UploadId;
685+
AbortMultipartUploadRequest abortRequest(BucketName, key, uploadId);
686+
auto abortOutcome = Client->AbortMultipartUpload(abortRequest);
687+
EXPECT_EQ(abortOutcome.isSuccess(), true);
688+
689+
// retry
690+
request.setFlags(request.Flags() ^ UploadPartFailedFlag);
691+
auto retryOutcome = Client->ResumableUploadObject(request);
692+
EXPECT_EQ(retryOutcome.isSuccess(), true);
693+
EXPECT_EQ(Client->DoesObjectExist(BucketName, key), true);
694+
695+
// download target object
696+
std::string targetFile = TestUtils::GetObjectKey("DownloadResumableUploadObject");
697+
auto getObjectOutcome = Client->GetObject(BucketName, key, targetFile);
698+
std::shared_ptr<std::iostream> getObjectContent = nullptr;
699+
getObjectOutcome.result().setContent(getObjectContent);
700+
EXPECT_EQ(getObjectOutcome.isSuccess(), true);
701+
702+
std::string targetFileMd5 = TestUtils::GetFileMd5(targetFile);
703+
EXPECT_EQ(targetFileMd5, sourceFileMd5);
704+
705+
EXPECT_EQ(RemoveFile(tmpFile), true);
706+
EXPECT_EQ(RemoveFile(targetFile), true);
707+
EXPECT_EQ(RemoveDirectory(checkpointKey), true);
708+
}
709+
658710
TEST_F(ResumableObjectTest, NormalResumableUploadWithProgressCallbackTest)
659711
{
660712
std::string key = TestUtils::GetObjectKey("NormalResumableUploadObjectWithCallback");
@@ -2074,6 +2126,45 @@ class ResumableObjectTest : public::testing::Test {
20742126
EXPECT_EQ(RemoveDirectory(checkpointDir), true);
20752127
}
20762128

2129+
2130+
TEST_F(ResumableObjectTest, NormalResumableCopyRetryWithUploadAbortTest)
2131+
{
2132+
std::string sourceKey = TestUtils::GetObjectKey("NormalResumableCopyRetryWithUploadAbortTest");
2133+
std::string targetKey = TestUtils::GetObjectKey("NormalResumableCopyRetryWithUploadAbortTest-target");
2134+
std::string checkpointDir = TestUtils::GetTargetFileName("checkpoint");
2135+
EXPECT_EQ(CreateDirectory(checkpointDir), true);
2136+
EXPECT_EQ(IsDirectoryExist(checkpointDir), true);
2137+
2138+
auto putObjectContent = TestUtils::GetRandomStream(102400 * (2 + rand() % 10));
2139+
auto putObjectOutcome = Client->PutObject(BucketName, sourceKey, putObjectContent);
2140+
EXPECT_EQ(putObjectOutcome.isSuccess(), true);
2141+
EXPECT_EQ(Client->DoesObjectExist(BucketName, sourceKey), true);
2142+
2143+
MultiCopyObjectRequest request(BucketName, targetKey, BucketName, sourceKey, checkpointDir, 102401, 1);
2144+
request.setFlags(request.Flags() | CopyPartFailedFlag);
2145+
auto outcome = Client->ResumableCopyObject(request);
2146+
EXPECT_EQ(outcome.isSuccess(), false);
2147+
EXPECT_EQ(Client->DoesObjectExist(BucketName, targetKey), false);
2148+
2149+
// abort upload Id
2150+
ListMultipartUploadsRequest lmuRequest(BucketName);
2151+
lmuRequest.setPrefix(targetKey);
2152+
auto lmuOutcome = Client->ListMultipartUploads(lmuRequest);
2153+
EXPECT_EQ(lmuOutcome.isSuccess(), true);
2154+
EXPECT_EQ(lmuOutcome.result().MultipartUploadList().size(), 1L);
2155+
auto uploadId = lmuOutcome.result().MultipartUploadList()[0].UploadId;
2156+
AbortMultipartUploadRequest abortRequest(BucketName, targetKey, uploadId);
2157+
auto abortOutcome = Client->AbortMultipartUpload(abortRequest);
2158+
EXPECT_EQ(abortOutcome.isSuccess(), true);
2159+
2160+
// retry
2161+
request.setFlags(request.Flags() ^ CopyPartFailedFlag);
2162+
auto retryOutcome = Client->ResumableCopyObject(request);
2163+
EXPECT_EQ(retryOutcome.isSuccess(), true);
2164+
EXPECT_EQ(Client->DoesObjectExist(BucketName, targetKey), true);
2165+
EXPECT_EQ(RemoveDirectory(checkpointDir), true);
2166+
}
2167+
20772168
TEST_F(ResumableObjectTest, NormalResumableCopyWithProgressCallbackTest)
20782169
{
20792170
std::string sourceKey = TestUtils::GetObjectKey("NormalDownloadSourceObjectWithProgressCallback");

0 commit comments

Comments
 (0)