Skip to content

Commit dc46281

Browse files
author
hero
committed
修改minio 文件分片上传，不够5M缓存
1 parent 152f698 commit dc46281

File tree

1 file changed

+121
-0
lines changed

1 file changed

+121
-0
lines changed

‎minio/main.go

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,90 @@
11
package main
22

33
import (
4+
"bytes"
45
"fmt"
6+
"github.com/go-redis/redis"
7+
"github.com/minio/minio-go"
58
"os"
9+
"time"
610
)
711

12+
var (
13+
minioCore *minio.Core
14+
redisClient *redis.Client
15+
err error
16+
)
17+
18+
func InitMinio() {
19+
minioCore, err = minio.NewCore("127.0.0.1:9000", "AKIAINCHU2DIYAQ66TPA", "ZaYtz0d61fJXQ7djyXBX4yZ5ob8Kj/WNXtw6PJob", false)
20+
CheckErr("minio.New", err)
21+
}
22+
23+
func InitRedis() {
24+
redisClient = redis.NewClient(&redis.Options{
25+
Addr: "127.0.0.1:6379",
26+
})
27+
pong, err := redisClient.Ping().Result()
28+
CheckErr(pong, err)
29+
}
30+
831
//分片上传文件
932
//分片文件小于5M 再redis缓存到5M在上传
1033
func main() {
34+
InitMinio()
35+
InitRedis()
36+
var (
37+
bucket = "test"
38+
name = "3.log"
39+
uploadId string
40+
completeParts []minio.CompletePart
41+
maxSize = 1024 * 1024 * 5
42+
)
1143

44+
uploadId, err = minioCore.NewMultipartUpload(bucket, name, minio.PutObjectOptions{})
45+
CheckErr("NewMultipartUpload", err)
46+
fmt.Println("uploadId", uploadId)
47+
var j = 1
48+
for i := 1; i < 20; i++ {
49+
//先判断文件大小是不是满足5M
50+
time.Sleep(time.Second * 3)
51+
a := bytes.Repeat([]byte(fmt.Sprint(1)), 1024*1024)
52+
redisA, err := redisClient.Get(uploadId).Bytes()
53+
if err != nil {
54+
if err != redis.Nil {
55+
fmt.Println("redisClient.Get", uploadId, err)
56+
panic(err)
57+
}
58+
}
59+
var data bytes.Buffer
60+
data.Write(redisA)
61+
data.Write(a)
62+
fmt.Println("index", i, " 文件大小", len(data.Bytes())/1024/1024)
63+
if len(data.Bytes()) < maxSize { //小于继续缓存
64+
if i >= 19 { //最后一片
65+
//最后一片直接上传
66+
fmt.Println("最后一片直接上传")
67+
} else {
68+
fmt.Println(uploadId, "内容太小继续缓存")
69+
redisClient.Set(uploadId, data.Bytes(), time.Minute*30)
70+
continue
71+
}
1272

73+
}
74+
part, err := minioCore.PutObjectPart(bucket, name, uploadId, j, bytes.NewReader(data.Bytes()), int64(len(data.Bytes())), "", "", nil)
75+
CheckErr("PutObjectPart", err)
76+
completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
77+
fmt.Println("uploadId :", uploadId, "part :", j, part)
78+
//该缓存已经上传完毕删除该内容
79+
fmt.Println("uploadId :", uploadId, redisClient.Del(uploadId).String())
80+
j++
81+
82+
}
83+
_, err = minioCore.CompleteMultipartUpload(bucket, name, uploadId, completeParts)
84+
if err != nil {
85+
panic(err)
86+
}
87+
fmt.Println("上传文件成功", uploadId)
1388
}
1489

1590
func CheckErr(msg string, err error) {
@@ -18,3 +93,49 @@ func CheckErr(msg string, err error) {
1893
os.Exit(1)
1994
}
2095
}
96+
97+
//uploadId 27ce1dec-f742-433e-a284-8eafeb22476b
98+
//index 1 文件大小 1
99+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
100+
//index 2 文件大小 2
101+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
102+
//index 3 文件大小 3
103+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
104+
//index 4 文件大小 4
105+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
106+
//index 5 文件大小 5
107+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b part : 1 {1 0001-01-01 00:00:00 +0000 UTC 7f0883388269a3eeb5b116b39ad81ee9-1 5242880}
108+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b del 27ce1dec-f742-433e-a284-8eafeb22476b: 1
109+
//index 6 文件大小 1
110+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
111+
//index 7 文件大小 2
112+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
113+
//index 8 文件大小 3
114+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
115+
//index 9 文件大小 4
116+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
117+
//index 10 文件大小 5
118+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b part : 2 {2 0001-01-01 00:00:00 +0000 UTC 4de9698076020ddd32de7230316d8ce5-1 5242880}
119+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b del 27ce1dec-f742-433e-a284-8eafeb22476b: 1
120+
//index 11 文件大小 1
121+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
122+
//index 12 文件大小 2
123+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
124+
//index 13 文件大小 3
125+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
126+
//index 14 文件大小 4
127+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
128+
//index 15 文件大小 5
129+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b part : 3 {3 0001-01-01 00:00:00 +0000 UTC 3717c6e04fe51d8028344be6059acef7-1 5242880}
130+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b del 27ce1dec-f742-433e-a284-8eafeb22476b: 1
131+
//index 16 文件大小 1
132+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
133+
//index 17 文件大小 2
134+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
135+
//index 18 文件大小 3
136+
//27ce1dec-f742-433e-a284-8eafeb22476b 内容太小继续缓存
137+
//index 19 文件大小 4
138+
//最后一片直接上传
139+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b part : 4 {4 0001-01-01 00:00:00 +0000 UTC e4c04a2710241b58b66fe51677acf3af-1 4194304}
140+
//uploadId : 27ce1dec-f742-433e-a284-8eafeb22476b del 27ce1dec-f742-433e-a284-8eafeb22476b: 1
141+
//上传文件成功 27ce1dec-f742-433e-a284-8eafeb22476b

0 commit comments

Comments
 (0)