From d75bc8f2befc96f8dd87490db3808d3afa164e2c Mon Sep 17 00:00:00 2001 From: Hiven Date: Wed, 2 Aug 2023 11:13:28 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/logic/uploadfilesbackendlogic.go | 89 ++++++++++++------- .../logic/uploadfilesfrontendlogic.go | 78 ++++++++++------ server/upload/internal/types/types.go | 8 +- server_api/upload.api | 8 +- utils/hash/hash.go | 70 +++++++++++++++ 5 files changed, 186 insertions(+), 67 deletions(-) create mode 100644 utils/hash/hash.go diff --git a/server/upload/internal/logic/uploadfilesbackendlogic.go b/server/upload/internal/logic/uploadfilesbackendlogic.go index 95b0bc86..bddf0cf2 100644 --- a/server/upload/internal/logic/uploadfilesbackendlogic.go +++ b/server/upload/internal/logic/uploadfilesbackendlogic.go @@ -1,9 +1,12 @@ package logic import ( + "encoding/json" + "fmt" "fusenapi/model/gmodel" "fusenapi/utils/auth" "fusenapi/utils/basic" + "fusenapi/utils/hash" "io" "net/http" "time" @@ -54,6 +57,7 @@ func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, } // 定义用户ID和S3键名格式 + var uid int64 var userId int64 var guestId int64 @@ -61,16 +65,21 @@ func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, if userinfo.IsGuest() { // 如果是，使用游客ID和游客键名格式 guestId = userinfo.GuestId + uid = guestId } else { // 否则，使用用户ID和用户键名格式 userId = userinfo.UserId + uid = userId } - var aa = make([]types.UploadInfo, 2) - aa[0].FileKeys = "202308011632" - aa[1].FileKeys = "202308011633" - req.UploadInfo = aa - var fileLen = len(req.UploadInfo) + var uploadInfoList []UploadInfo + err := json.Unmarshal([]byte(req.UploadInfo), &uploadInfoList) + if err != nil { + logx.Error(err) + return resp.SetStatus(basic.CodeFileUploadErr, "file upload err,params Unmarshal failed") + } + + var fileLen = len(uploadInfoList) if fileLen == 0 { return resp.SetStatus(basic.CodeFileUploadErr, "file upload err,no files") @@ -78,6 +87,18 @@ func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, if req.ApiType == 1 && fileLen > 100 { return resp.SetStatus(basic.CodeFileUploadErr, "file upload err, files count is beyond the maximum") } + + // 定义存储桶名称 + var bucketName *string + + // 根据类别选择存储桶 + switch req.UploadBucket { + case 2: + bucketName = basic.TempfileBucketName + default: + bucketName = basic.StorageBucketName + } + //设置内存大小 l.r.ParseMultipartForm(32 << 20) @@ -93,10 +114,9 @@ func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, // 定义S3请求和当前时间 var s3req *request.Request - var uploadBucket = req.UploadBucket resourceModel := gmodel.NewFsResourceModel(l.svcCtx.MysqlConn) result, err := mr.MapReduce(func(source chan<- interface{}) { - for i, info := range req.UploadInfo { + for i, info := range uploadInfoList { fileType := files[i].Header.Get("Content-Type") // 打开文件 file, err := files[i].Open() @@ -111,34 +131,36 @@ func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, } // 一系列业务逻辑....验证类型,文件大小 - var fileKey string = info.FileKeys - source <- uploadData{ - FileKey: fileKey, + + var hashKey string = hash.JsonHashKey(fmt.Sprintf("%s%d", info.FileKeys, uid)) + source <- UploadData{ + FileKey: info.FileKeys, FileType: fileType, Metadata: info.Metadata, FileData: ioData, ApiType: req.ApiType, - Bucket: uploadBucket, + Bucket: bucketName, + HashKey: hashKey, } } }, func(item interface{}, writer mr.Writer[interface{}], cancel func(error)) { - var uploadUrl = uploadUrl{} - uploadDataInfo := item.(uploadData) + var uploadUrl = UploadUrl{} + uploadDataInfo := item.(UploadData) - var resourceId string = uploadDataInfo.FileKey + var resourceId string = uploadDataInfo.HashKey // 查询数据库 resourceInfo, err := resourceModel.FindOneById(l.ctx, resourceId) if err == nil && resourceInfo.ResourceId != "" { uploadUrl.Status = 1 uploadUrl.ResourceId = resourceId uploadUrl.Url = *resourceInfo.ResourceUrl - uploadUrl.Key = resourceId + uploadUrl.Key = uploadDataInfo.FileKey } else { // 创建S3对象存储请求 s3req, _ = svc.PutObjectRequest( &s3.PutObjectInput{ - Bucket: &uploadDataInfo.Bucket, - Key: &uploadDataInfo.FileKey, + Bucket: uploadDataInfo.Bucket, + Key: &uploadDataInfo.HashKey, }, ) @@ -184,11 +206,11 @@ func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, // Notice 这个必须加! writer.Write(uploadUrl) }, func(pipe <-chan interface{}, writer mr.Writer[interface{}], cancel func(error)) { - var uploadUrlList = make(map[string][]*uploadUrl) - var uploadUrlListFail []*uploadUrl - var uploadUrlListSuccess []*uploadUrl + var uploadUrlList = make(map[string][]*UploadUrl) + var uploadUrlListFail []*UploadUrl + var uploadUrlListSuccess []*UploadUrl for p := range pipe { - var uploadUrl = p.(uploadUrl) + var uploadUrl = p.(UploadUrl) if uploadUrl.Status == 1 { uploadUrlListSuccess = append(uploadUrlListSuccess, &uploadUrl) } else { @@ -211,17 +233,24 @@ func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, }) } -type uploadData struct { - ApiType int64 `json:"api_type"` - FileSize int64 `json:"file_size"` - FileType string `json:"file_type"` - FileKey string `json:"file_key"` - Metadata string `json:"metadata"` - Bucket string `json:"bucket"` - FileData []byte `fsfile:"data"` +type UploadInfo struct { + FileSize int64 `json:"file_size"` // 上传唯一标识信息 + FileKeys string `json:"file_keys"` // 上传唯一标识信息 + Metadata string `json:"meta_data"` // 上传文件额外信息 } -type uploadUrl struct { +type UploadData struct { + ApiType int64 `json:"api_type"` + FileSize int64 `json:"file_size"` + FileType string `json:"file_type"` + FileKey string `json:"file_key"` + Metadata string `json:"metadata"` + Bucket *string `json:"bucket"` + HashKey string `json:"hash_key"` + FileData []byte `fsfile:"data"` +} + +type UploadUrl struct { Status int64 `json:"status"` ResourceId string `json:"resource_id"` Url string `json:"url"` diff --git a/server/upload/internal/logic/uploadfilesfrontendlogic.go b/server/upload/internal/logic/uploadfilesfrontendlogic.go index 1edabc89..b0910736 100644 --- a/server/upload/internal/logic/uploadfilesfrontendlogic.go +++ b/server/upload/internal/logic/uploadfilesfrontendlogic.go @@ -1,8 +1,11 @@ package logic import ( + "encoding/json" + "fmt" "fusenapi/utils/auth" "fusenapi/utils/basic" + "fusenapi/utils/hash" "time" "context" @@ -42,25 +45,30 @@ func NewUploadFilesFrontendLogic(ctx context.Context, svcCtx *svc.ServiceContext func (l *UploadFilesFrontendLogic) UploadFilesFrontend(req *types.UploadFilesReq, userinfo *auth.UserInfo) (resp *basic.Response) { // 返回值必须调用Set重新返回, resp可以空指针调用 resp.SetStatus(basic.CodeOK, data) // userinfo 传入值时, 一定不为null - // 定义用户ID和S3键名格式 - // var userId int64 - // var guestId int64 + var uid int64 + var userId int64 + var guestId int64 // 检查用户是否是游客 - // if userinfo.IsGuest() { - // // 如果是，使用游客ID和游客键名格式 - // guestId = userinfo.GuestId - // } else { - // // 否则，使用用户ID和用户键名格式 - // userId = userinfo.UserId - // } + if userinfo.IsGuest() { + // 如果是，使用游客ID和游客键名格式 + guestId = userinfo.GuestId + uid = guestId + } else { + // 否则，使用用户ID和用户键名格式 + userId = userinfo.UserId + uid = userId + } - var aa = make([]types.UploadInfo, 2) - aa[0].FileKeys = "202308011632" - aa[1].FileKeys = "202308011633" - req.UploadInfo = aa - var fileLen = len(req.UploadInfo) + var uploadInfoList []UploadInfo + err := json.Unmarshal([]byte(req.UploadInfo), &uploadInfoList) + if err != nil { + logx.Error(err) + return resp.SetStatus(basic.CodeFileUploadErr, "file upload err,params Unmarshal failed") + } + + var fileLen = len(uploadInfoList) if fileLen == 0 { return resp.SetStatus(basic.CodeFileUploadErr, "file upload err,no files") @@ -69,7 +77,16 @@ func (l *UploadFilesFrontendLogic) UploadFilesFrontend(req *types.UploadFilesReq return resp.SetStatus(basic.CodeFileUploadErr, "file upload err, files count is beyond the maximum") } - var uploadBucket = req.UploadBucket + // 定义存储桶名称 + var bucketName *string + + // 根据类别选择存储桶 + switch req.UploadBucket { + case 2: + bucketName = basic.TempfileBucketName + default: + bucketName = basic.StorageBucketName + } // 设置AWS会话的区域 l.svcCtx.AwsSession.Config.Region = aws.String("us-west-1") @@ -78,25 +95,26 @@ func (l *UploadFilesFrontendLogic) UploadFilesFrontend(req *types.UploadFilesReq svc := s3.New(l.svcCtx.AwsSession) result, err := mr.MapReduce(func(source chan<- interface{}) { - for _, info := range req.UploadInfo { + for _, info := range uploadInfoList { if info.FileSize <= 1024*1024*500 { // 一系列业务逻辑....验证类型,文件大小 - var fileKey string = info.FileKeys - source <- uploadData{ - FileKey: fileKey, + var hashKey string = hash.JsonHashKey(fmt.Sprintf("%s%d", info.FileKeys, uid)) + source <- UploadData{ + FileKey: info.FileKeys, FileSize: info.FileSize, - Bucket: uploadBucket, + Bucket: bucketName, + HashKey: hashKey, } } } }, func(item interface{}, writer mr.Writer[interface{}], cancel func(error)) { - var uploadUrl = uploadUrl{} - uploadDataInfo := item.(uploadData) + var uploadUrl = UploadUrl{} + uploadDataInfo := item.(UploadData) s3req, _ := svc.PutObjectRequest( &s3.PutObjectInput{ - Bucket: &uploadBucket, - Key: &uploadDataInfo.FileKey, + Bucket: uploadDataInfo.Bucket, + Key: &uploadDataInfo.HashKey, ContentLength: aws.Int64(uploadDataInfo.FileSize), }, ) @@ -106,22 +124,24 @@ func (l *UploadFilesFrontendLogic) UploadFilesFrontend(req *types.UploadFilesReq logx.Error(err) uploadUrl.Status = 0 uploadUrl.Url = "" + uploadUrl.ResourceId = uploadDataInfo.HashKey uploadUrl.Key = uploadDataInfo.FileKey } else { // 打印请求URL logx.Info(url) uploadUrl.Status = 1 uploadUrl.Url = url + uploadUrl.ResourceId = uploadDataInfo.HashKey uploadUrl.Key = uploadDataInfo.FileKey } // Notice 这个必须加! writer.Write(uploadUrl) }, func(pipe <-chan interface{}, writer mr.Writer[interface{}], cancel func(error)) { - var uploadUrlList = make(map[string][]*uploadUrl) - var uploadUrlListFail []*uploadUrl - var uploadUrlListSuccess []*uploadUrl + var uploadUrlList = make(map[string][]*UploadUrl) + var uploadUrlListFail []*UploadUrl + var uploadUrlListSuccess []*UploadUrl for p := range pipe { - var uploadUrl = p.(uploadUrl) + var uploadUrl = p.(UploadUrl) if uploadUrl.Status == 1 { uploadUrlListSuccess = append(uploadUrlListSuccess, &uploadUrl) } else { diff --git a/server/upload/internal/types/types.go b/server/upload/internal/types/types.go index 5f8d2aa6..5a7a1a3f 100644 --- a/server/upload/internal/types/types.go +++ b/server/upload/internal/types/types.go @@ -8,13 +8,13 @@ import ( type UploadInfo struct { FileSize int64 `form:"file_size,optional"` // 上传唯一标识信息 FileKeys string `form:"file_keys,optional"` // 上传唯一标识信息 - Metadata string `form:"file_keys,optional"` // 上传文件额外信息 + Metadata string `form:"meta_data,optional"` // 上传文件额外信息 } type UploadFilesReq struct { - ApiType int64 `form:"api_type,options=[1,2],default=1"` // 调用类型：1=对外，2=对内 - UploadBucket string `form:"upload_bucket"` // 上传桶名 - UploadInfo []UploadInfo `form:"upload_info,optional"` // 上传信息 + ApiType int64 `form:"api_type,options=[1,2],default=1"` // 调用类型：1=对外，2=对内 + UploadBucket int64 `form:"upload_bucket,options=[1,2],default=1"` // 上传桶名:1=缓存,2=持久 + UploadInfo string `form:"upload_info"` // 上传信息 json } type UploadCallbackReq struct { diff --git a/server_api/upload.api b/server_api/upload.api index b10ae1b7..eca41ebf 100644 --- a/server_api/upload.api +++ b/server_api/upload.api @@ -39,13 +39,13 @@ type ( UploadInfo { FileSize int64 `form:"file_size,optional"` // 上传唯一标识信息 FileKeys string `form:"file_keys,optional"` // 上传唯一标识信息 - Metadata string `form:"file_keys,optional"` // 上传文件额外信息 + Metadata string `form:"meta_data,optional"` // 上传文件额外信息 } UploadFilesReq { - ApiType int64 `form:"api_type,options=[1,2],default=1"` // 调用类型：1=对外，2=对内 - UploadBucket string `form:"upload_bucket"` // 上传桶名 - UploadInfo []UploadInfo `form:"upload_info,optional"` // 上传信息 + ApiType int64 `form:"api_type,options=[1,2],default=1"` // 调用类型：1=对外，2=对内 + UploadBucket int64 `form:"upload_bucket,options=[1,2],default=1"` // 上传桶名:1=缓存,2=持久 + UploadInfo string `form:"upload_info"` // 上传信息 json } UploadCallbackReq { FileType string `form:"file_type"` // 文件类型 / fbx / hdr diff --git a/utils/hash/hash.go b/utils/hash/hash.go new file mode 100644 index 00000000..37ea1c00 --- /dev/null +++ b/utils/hash/hash.go @@ -0,0 +1,70 @@ +package hash + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "sort" +) + +func JsonHashKey(v interface{}) string { + h := sha256.New() + h.Write(marshalOrdered(v)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func marshalOrdered(v interface{}) []byte { + switch v := v.(type) { + case map[string]interface{}: + sortedKeys := make([]string, 0, len(v)) + for key := range v { + sortedKeys = append(sortedKeys, key) + } + sort.Strings(sortedKeys) + + var buf bytes.Buffer + buf.WriteByte('{') + for i, key := range sortedKeys { + if i > 0 { + buf.WriteByte(',') + } + b, err := json.Marshal(key) + if err != nil { + panic(err) + } + buf.Write(b) + buf.WriteByte(':') + + b = marshalOrdered(v[key]) + buf.Write(b) + } + buf.WriteByte('}') + return buf.Bytes() + + case []interface{}: + var buf bytes.Buffer + + sort.Slice(v, func(i, j int) bool { + return bytes.Compare(marshalOrdered(v[i]), marshalOrdered(v[j])) == 1 + }) + buf.WriteByte('[') + for i, val := range v { + if i > 0 { + buf.WriteByte(',') + } + + b := marshalOrdered(val) + buf.Write(b) + } + buf.WriteByte(']') + return buf.Bytes() + + default: + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + return b + } +}