From 7818b88b70e84cb0043d7b556adf16ac362ea995 Mon Sep 17 00:00:00 2001 From: eson <9673575+githubcontent@user.noreply.gitee.com> Date: Fri, 7 Jul 2023 17:49:12 +0800 Subject: [PATCH] =?UTF-8?q?upload=20=E4=B8=8A=E4=BC=A0=20=E7=AD=BE?= =?UTF-8?q?=E5=90=8D=20=E7=9B=B4=E6=8E=A5=E4=B8=8A=E4=BC=A0=E9=83=BD?= =?UTF-8?q?=E5=8F=AF=E4=BB=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 4 +- go.sum | 4 +- .../internal/logic/uploadfilebackendlogic.go | 44 ++++----------- .../internal/logic/uploadfilefrontendlogic.go | 29 ++++++---- .../test/uploadfilebackendhandler_test.go | 28 +++++++--- .../test/uploadfilefrontendhandler_test.go | 56 +++++++++++++++++++ utils/auth/user.go | 2 +- utils/basic/basic.go | 3 + utils/format/s3keyname.go | 16 +++--- 9 files changed, 120 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index 7190e595..ad5f7bb6 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go v1.44.295 github.com/bwmarrin/snowflake v0.3.0 github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/google/uuid v1.3.0 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/stripe/stripe-go/v74 v74.22.0 @@ -18,13 +19,12 @@ require ( require ( github.com/andybalholm/brotli v1.0.5 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/schollz/progressbar v1.0.0 // indirect ) require ( - github.com/474420502/requests v1.36.0 + github.com/474420502/requests v1.38.0 github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/tidwall/gjson v1.12.0 diff --git a/go.sum b/go.sum index e8b9a6b7..1a2cf05a 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/474420502/random v0.4.1 h1:HUUyLXRWMijVb7CJoEC16f0aFQOW25Lkr80Mut6PoKU= -github.com/474420502/requests v1.36.0 h1:oaIRAoi8vhPwdR3AOnYUsYMMN+2Mi1rI9r/fvXA3wiE= -github.com/474420502/requests v1.36.0/go.mod h1:r1Uwbbq/t3Cn95VOfgJjZmD7sfwp+gjldq/zDncNYRc= +github.com/474420502/requests v1.38.0 h1:ASIS9i9k0njA8lH8XFm4jssrnz+10smu4rFkehLen5w= +github.com/474420502/requests v1.38.0/go.mod h1:r1Uwbbq/t3Cn95VOfgJjZmD7sfwp+gjldq/zDncNYRc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= diff --git a/server/upload/internal/logic/uploadfilebackendlogic.go b/server/upload/internal/logic/uploadfilebackendlogic.go index b0aee8fb..34f74397 100644 --- a/server/upload/internal/logic/uploadfilebackendlogic.go +++ b/server/upload/internal/logic/uploadfilebackendlogic.go @@ -5,7 +5,6 @@ import ( "fusenapi/utils/basic" "fusenapi/utils/check" "fusenapi/utils/format" - "log" "time" "context" @@ -69,50 +68,27 @@ func (l *UploadFileBackendLogic) UploadFileBackend(req *types.RequestUploadFileB req.File.Filename, )) + var bucketName *string switch category { case format.TCategoryRenderMegre: - - lifecycleConfiguration := &s3.BucketLifecycleConfiguration{ - Rules: []*s3.LifecycleRule{ - { - Status: aws.String("Enabled"), - // Status: aws.String("Disabled"), - Filter: &s3.LifecycleRuleFilter{ - Prefix: ObjectKey, - }, - Expiration: &s3.LifecycleExpiration{ - // Date: aws.Time(time.Now().UTC().Add(time.Minute)), - Days: aws.Int64(1), // 设置过期天数，例如：30天 - }, - // ID: aws.String("ExpireAfter01Days"), // 设置规则的唯一标识符 - }, - }, - } - - s3req, _ = svc.PutBucketLifecycleConfigurationRequest( - &s3.PutBucketLifecycleConfigurationInput{ - Bucket: basic.StorageBucketName, - LifecycleConfiguration: lifecycleConfiguration, - }, - ) + bucketName = basic.TempfileBucketName default: - s3req, _ = svc.PutObjectRequest( - &s3.PutObjectInput{ - Bucket: basic.StorageBucketName, - Key: ObjectKey, - }, - ) - + bucketName = basic.StorageBucketName } + s3req, _ = svc.PutObjectRequest( + &s3.PutObjectInput{ + Bucket: bucketName, + Key: ObjectKey, + }, + ) s3req.SetBufferBody(req.File.Data) err := s3req.Send() if err != nil { logx.Error(err) return resp.SetStatus(basic.CodeS3PutObjectRequestErr) } - log.Println(s3req.HTTPRequest.URL) - + logx.Info(s3req.HTTPRequest.URL.String()) return resp.SetStatus(basic.CodeOK, map[string]interface{}{ "upload_url": s3req.HTTPRequest.URL.String(), }) diff --git a/server/upload/internal/logic/uploadfilefrontendlogic.go b/server/upload/internal/logic/uploadfilefrontendlogic.go index 847fbc4e..7789c86f 100644 --- a/server/upload/internal/logic/uploadfilefrontendlogic.go +++ b/server/upload/internal/logic/uploadfilefrontendlogic.go @@ -61,18 +61,28 @@ func (l *UploadFileFrontendLogic) UploadFileFrontend(req *types.RequestUploadFil } now := time.Now() + category := format.TypeCategory(req.Category) + ObjectKey := aws.String(format.FormatS3KeyName( + keytype, + uid, + now, + l.svcCtx.Config.Env, + format.TypeCategory(req.Category), + req.FileName, + )) + + var bucketName *string + switch category { + case format.TCategoryRenderMegre: + bucketName = basic.TempfileBucketName + default: + bucketName = basic.StorageBucketName + } s3req, _ := svc.PutObjectRequest( &s3.PutObjectInput{ - Bucket: aws.String("storage.fusenpack.com"), - Key: aws.String(format.FormatS3KeyName( - keytype, - uid, - now, - l.svcCtx.Config.Env, - format.TypeCategory(req.Category), - req.FileName, - )), + Bucket: bucketName, + Key: ObjectKey, ContentLength: aws.Int64(req.FileSize), }, ) @@ -81,7 +91,6 @@ func (l *UploadFileFrontendLogic) UploadFileFrontend(req *types.RequestUploadFil if err != nil { return resp.SetStatus(basic.CodeS3PutObjectRequestErr) } - // log.Println(uri) return resp.SetStatus(basic.CodeOK, map[string]interface{}{ "upload_url": uri, diff --git a/server/upload/test/uploadfilebackendhandler_test.go b/server/upload/test/uploadfilebackendhandler_test.go index 2baf73cb..c7828499 100644 --- a/server/upload/test/uploadfilebackendhandler_test.go +++ b/server/upload/test/uploadfilebackendhandler_test.go @@ -4,6 +4,8 @@ import ( "fmt" "fusenapi/utils/format" fstests "fusenapi/utils/fstests" + "log" + "os" "testing" "github.com/474420502/requests" @@ -19,16 +21,24 @@ func TestCaseRenderMegre(t *testing.T) { ses := fstests.GetSessionWithUserToken(t, userver, cnf.Host, cnf.Port) tp := ses.Post(fmt.Sprintf("http://%s:%d/upload/upload-file-backend", cnf.Host, cnf.Port)) - // data, err := ioutil.ReadFile("./fusen.webp") - // if err != nil { - // t.Error(err) - // return - // } - mp := tp.CreateBodyMultipart() - mp.WriteField("file_type", "image") - mp.WriteField("category", string(format.TCategoryRenderMegre)) - mp.CreateFormFile("file", "./fusen.web") + f, err := os.Open("./fusen.webp") + if err != nil { + panic(err) + } + + err = mp.AddFieldFile("file", f.Name(), f) + if err != nil { + log.Println(err) + } + err = mp.AddField("file_type", "image") + if err != nil { + panic(err) + } + err = mp.AddField("category", string(format.TCategoryRenderMegre)) + if err != nil { + panic(err) + } tp.SetBodyFormData(mp) // 向服务器发送 GET 请求，获取用户基本信息 diff --git a/server/upload/test/uploadfilefrontendhandler_test.go b/server/upload/test/uploadfilefrontendhandler_test.go index 5cab7f85..8564c48f 100644 --- a/server/upload/test/uploadfilefrontendhandler_test.go +++ b/server/upload/test/uploadfilefrontendhandler_test.go @@ -3,6 +3,7 @@ package test import ( "fmt" "fusenapi/server/upload/internal/types" + "fusenapi/utils/format" fstests "fusenapi/utils/fstests" "io/ioutil" "log" @@ -66,3 +67,58 @@ func TestCasePersonalization(t *testing.T) { t.Error("put failed") } } + +func TestCaseRenderMegreF(t *testing.T) { + var err error + var resp *requests.Response + var result gjson.Result + + // 获取 session，并携带 JWT token + ses := fstests.GetSessionWithUserToken(t, userver, cnf.Host, cnf.Port) + tp := ses.Post(fmt.Sprintf("http://%s:%d/upload/upload-file-frontend", cnf.Host, cnf.Port)) + + data, err := ioutil.ReadFile("./fusen.webp") + if err != nil { + t.Error(err) + return + } + + req := types.RequestUploadFileFrontend{ + FileName: "fusen.webp", FileType: "image", FileSize: int64(len(data)), + Category: string(format.TCategoryRenderMegre), + } + tp.SetBodyJson(req) + // 向服务器发送 GET 请求，获取用户基本信息 + resp, err = tp.TestExecute(gserver) + if err != nil { + t.Error(err) + } + + result = resp.Json() + if result.Get("code").Int() != 200 { + t.Error("code != 200") + } + + if !result.Get("data").Exists() { + t.Error("data not exists") + } + + if !result.Get("data.upload_url").Exists() { + t.Error("upload_url not exists") + } + + log.Println(result) + uri := result.Get("data.upload_url").String() + + tpUpload := requests.NewSession().Put(uri) + tpUpload.SetBodyStream(data) + + resp, err = tpUpload.Execute() + if err != nil { + t.Error(err) + } + + if resp.GetStatusCode() != 200 { + t.Error("put failed") + } +} diff --git a/utils/auth/user.go b/utils/auth/user.go index 9bdb6cb2..3acb8249 100644 --- a/utils/auth/user.go +++ b/utils/auth/user.go @@ -60,7 +60,7 @@ func (info *UserInfo) IsGuest() bool { // IsOnlooker 白板用户: 非登录用户, 非游客, 判断为白板用户 func (info *UserInfo) IsOnlooker() bool { - return info.UserId != 0 && info.GuestId != 0 + return info.UserId == 0 && info.GuestId == 0 } type BackendUserInfo struct { diff --git a/utils/basic/basic.go b/utils/basic/basic.go index 5282ef47..0e7e2e5b 100644 --- a/utils/basic/basic.go +++ b/utils/basic/basic.go @@ -13,6 +13,9 @@ import ( // 全局的BucketName var StorageBucketName = aws.String("storage.fusenpack.com") +// 全局的BucketName 缓存使用 +var TempfileBucketName = aws.String("tempfile.fusenpack.com") + const UploadFileLimitSize = 200 << 20 // File uploadfile 文件(multipart...) diff --git a/utils/format/s3keyname.go b/utils/format/s3keyname.go index 8765e0c8..b5be968a 100644 --- a/utils/format/s3keyname.go +++ b/utils/format/s3keyname.go @@ -56,15 +56,15 @@ func FormatS3KeyNameUser(userid int64, now time.Time, env string, category TypeC return fmt.Sprintf("/%s/%s/%d/%s.%s", env, category, userid, name, ext) case TCategoryRenderMegre: // /{env}/render_megre/{userid}/{filename} 自动删除 - return fmt.Sprintf("/%s/%s/%d/%s_%d.%s", env, category, userid, name, now.Unix(), ext) + return fmt.Sprintf("/%s/%s/%d/%s_%d.%s", env, category, userid, name, now.UnixNano(), ext) case TCategory3DTools: // /{env}/3dtools/年月/{userid}/{filename} - return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), userid, name, now.Unix(), ext) + return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), userid, name, now.UnixNano(), ext) case TCategoryQuotation: // /{env}/quotation/年月/{userid}/{filename} - return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), userid, name, now.Unix(), ext) + return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), userid, name, now.UnixNano(), ext) default: - return fmt.Sprintf("/%s/%s/%d/%04d%02d/%s_%d.%s", env, category, userid, year, int(month), name, now.Unix(), ext) + return fmt.Sprintf("/%s/%s/%d/%04d%02d/%s_%d.%s", env, category, userid, year, int(month), name, now.UnixNano(), ext) } } @@ -87,17 +87,17 @@ func FormatS3KeyNameGuest(guestid int64, now time.Time, env string, category Typ return fmt.Sprintf("/%s/%s/%d/%s.%s", env, category, guestid, name, ext) case TCategoryRenderMegre: // /{env}/render_megre/{guestid}/{filename} 自动删除 - return fmt.Sprintf("/%s/%s/%d/%s_%d.%s", env, category, guestid, name, now.Unix(), ext) + return fmt.Sprintf("/%s/%s/%d/%s_%d.%s", env, category, guestid, name, now.UnixNano(), ext) case TCategory3DTools: panic(TCategory3DTools + "不存在游客") // /{env}/3dtools/年月/{guestid}/{filename} - // return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), guestid, name, now.Unix(), ext) + // return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), guestid, name, now.UnixNano(), ext) case TCategoryQuotation: panic(TCategoryQuotation + "不存在游客") // /{env}/quotation/年月/{guestid}/{filename} - // return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), guestid, name, now.Unix(), ext) + // return fmt.Sprintf("/%s/%s/%04d%02d/%d/%s_%d.%s", env, category, year, int(month), guestid, name, now.UnixNano(), ext) default: - return fmt.Sprintf("/%s/%s/%d/%04d%02d/%s_%d.%s", env, category, guestid, year, int(month), name, now.Unix(), ext) + return fmt.Sprintf("/%s/%s/%d/%04d%02d/%s_%d.%s", env, category, guestid, year, int(month), name, now.UnixNano(), ext) } }