多文件上传--后端

This commit is contained in:
Hiven 2023-08-01 18:13:30 +08:00
parent c75e55a5cf
commit 4a2a7822b7
14 changed files with 644 additions and 4 deletions

View File

@ -16,6 +16,7 @@ type FsResource struct {
UploadedAt *time.Time `gorm:"index;default:'0000-00-00 00:00:00';" json:"uploaded_at"` // 上传时间
Metadata *string `gorm:"default:'';" json:"metadata"` // 元数据,json格式,存储图像分率
MetaKey1 *string `gorm:"index;default:'';" json:"meta_key1"` // 需要关键信息查询的自定义属性1,可以动态增加
ApiType *int64 `gorm:"default:1;" json:"api_type"` // 调用类型1=对外2=对内
}
type FsResourceModel struct {
db *gorm.DB

View File

@ -1,2 +1,84 @@
package gmodel
// TODO: 使用model的属性做你想做的
import (
"context"
"errors"
"fusenapi/utils/handler"
"gorm.io/gorm"
)
// TODO: 使用model的属性做你想做的
func (p *FsResourceModel) FindOneById(ctx context.Context, resourceId string) (*FsResource, error) {
var resp FsResource
result := p.db.Table(p.name).WithContext(ctx).Where("resource_id =?", resourceId).Take(&resp)
if result.Error != nil {
// 检查 ErrRecordNotFound 错误
if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
return nil, result.Error
}
}
return &resp, nil
}
func (p *FsResourceModel) CreateOrUpdate(ctx context.Context, req *FsResource) (resp *FsResource, err error) {
rowBuilder := p.db.Table(p.name).WithContext(ctx)
if req.ResourceId != "" {
err = rowBuilder.Save(req).Error
} else {
err = rowBuilder.Create(req).Error
}
return req, err
}
func (m *FsResourceModel) FindOneByQuery(ctx context.Context, rowBuilder *gorm.DB, filterMap map[string]string) (*FsResource, error) {
var resp FsResource
if filterMap != nil {
rowBuilder = rowBuilder.Scopes(handler.FilterData(filterMap))
}
result := rowBuilder.WithContext(ctx).Limit(1).Find(&resp)
if result.Error != nil {
return nil, result.Error
} else {
return &resp, nil
}
}
func (m *FsResourceModel) RowSelectBuilder(selectData []string) *gorm.DB {
var rowBuilder = m.db.Table(m.name)
if selectData != nil {
rowBuilder = rowBuilder.Select(selectData)
} else {
rowBuilder = rowBuilder.Select("*")
}
return rowBuilder
}
// 事务
func (m *FsResourceModel) Trans(ctx context.Context, fn func(ctx context.Context, connGorm *gorm.DB) error) error {
tx := m.db.Table(m.name).WithContext(ctx).Begin()
defer func() {
if r := recover(); r != nil {
tx.Rollback()
}
}()
if err := tx.Error; err != nil {
return err
}
if err := fn(ctx, tx); err != nil {
tx.Rollback()
return err
}
return tx.Commit().Error
}
func (m *FsResourceModel) TableName() string {
return m.name
}

View File

@ -1,2 +1,71 @@
package gmodel
// TODO: 使用model的属性做你想做的
import (
"context"
"fusenapi/utils/handler"
"gorm.io/gorm"
)
// TODO: 使用model的属性做你想做的
func (p *FsResourcesModel) CreateOrUpdate(ctx context.Context, req *FsResources) (resp *FsResources, err error) {
rowBuilder := p.db.Table(p.name).WithContext(ctx)
if req.ResourceId != "" {
err = rowBuilder.Save(req).Error
} else {
err = rowBuilder.Create(req).Error
}
return req, err
}
func (m *FsResourcesModel) FindOneByQuery(ctx context.Context, rowBuilder *gorm.DB, filterMap map[string]string) (*FsResources, error) {
var resp FsResources
if filterMap != nil {
rowBuilder = rowBuilder.Scopes(handler.FilterData(filterMap))
}
result := rowBuilder.WithContext(ctx).Limit(1).Find(&resp)
if result.Error != nil {
return nil, result.Error
} else {
return &resp, nil
}
}
func (m *FsResourcesModel) RowSelectBuilder(selectData []string) *gorm.DB {
var rowBuilder = m.db.Table(m.name)
if selectData != nil {
rowBuilder = rowBuilder.Select(selectData)
} else {
rowBuilder = rowBuilder.Select("*")
}
return rowBuilder
}
// 事务
func (m *FsResourcesModel) Trans(ctx context.Context, fn func(ctx context.Context, connGorm *gorm.DB) error) error {
tx := m.db.Table(m.name).WithContext(ctx).Begin()
defer func() {
if r := recover(); r != nil {
tx.Rollback()
}
}()
if err := tx.Error; err != nil {
return err
}
if err := fn(ctx, tx); err != nil {
tx.Rollback()
return err
}
return tx.Commit().Error
}
func (m *FsResourcesModel) TableName() string {
return m.name
}

View File

@ -32,6 +32,21 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) {
Path: "/api/upload/qrcode",
Handler: UploadQrcodeHandler(serverCtx),
},
{
Method: http.MethodPost,
Path: "/api/upload/upload-files-backend",
Handler: UploadFilesBackendHandler(serverCtx),
},
{
Method: http.MethodPost,
Path: "/api/upload/upload-files-frontend",
Handler: UploadFilesFrontendHandler(serverCtx),
},
{
Method: http.MethodPost,
Path: "/api/upload/upload-callback",
Handler: UploadCallbackHandler(serverCtx),
},
},
)
}

View File

@ -0,0 +1,35 @@
package handler
import (
"net/http"
"reflect"
"fusenapi/utils/basic"
"fusenapi/server/upload/internal/logic"
"fusenapi/server/upload/internal/svc"
"fusenapi/server/upload/internal/types"
)
func UploadCallbackHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req types.UploadCallbackReq
userinfo, err := basic.RequestParse(w, r, svcCtx, &req)
if err != nil {
return
}
// 创建一个业务逻辑层实例
l := logic.NewUploadCallbackLogic(r.Context(), svcCtx)
rl := reflect.ValueOf(l)
basic.BeforeLogic(w, r, rl)
resp := l.UploadCallback(&req, userinfo)
if !basic.AfterLogic(w, r, rl, resp) {
basic.NormalAfterLogic(w, r, resp)
}
}
}

View File

@ -0,0 +1,35 @@
package handler
import (
"net/http"
"reflect"
"fusenapi/utils/basic"
"fusenapi/server/upload/internal/logic"
"fusenapi/server/upload/internal/svc"
"fusenapi/server/upload/internal/types"
)
func UploadFilesBackendHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req types.UploadFilesReq
userinfo, err := basic.RequestParse(w, r, svcCtx, &req)
if err != nil {
return
}
// 创建一个业务逻辑层实例
l := logic.NewUploadFilesBackendLogic(r, svcCtx)
rl := reflect.ValueOf(l)
basic.BeforeLogic(w, r, rl)
resp := l.UploadFilesBackend(&req, userinfo)
if !basic.AfterLogic(w, r, rl, resp) {
basic.NormalAfterLogic(w, r, resp)
}
}
}

View File

@ -0,0 +1,35 @@
package handler
import (
"net/http"
"reflect"
"fusenapi/utils/basic"
"fusenapi/server/upload/internal/logic"
"fusenapi/server/upload/internal/svc"
"fusenapi/server/upload/internal/types"
)
func UploadFilesFrontendHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req types.UploadFilesReq
userinfo, err := basic.RequestParse(w, r, svcCtx, &req)
if err != nil {
return
}
// 创建一个业务逻辑层实例
l := logic.NewUploadFilesFrontendLogic(r.Context(), svcCtx)
rl := reflect.ValueOf(l)
basic.BeforeLogic(w, r, rl)
resp := l.UploadFilesFrontend(&req, userinfo)
if !basic.AfterLogic(w, r, rl, resp) {
basic.NormalAfterLogic(w, r, resp)
}
}
}

View File

@ -0,0 +1,42 @@
package logic
import (
"fusenapi/utils/auth"
"fusenapi/utils/basic"
"context"
"fusenapi/server/upload/internal/svc"
"fusenapi/server/upload/internal/types"
"github.com/zeromicro/go-zero/core/logx"
)
type UploadCallbackLogic struct {
logx.Logger
ctx context.Context
svcCtx *svc.ServiceContext
}
func NewUploadCallbackLogic(ctx context.Context, svcCtx *svc.ServiceContext) *UploadCallbackLogic {
return &UploadCallbackLogic{
Logger: logx.WithContext(ctx),
ctx: ctx,
svcCtx: svcCtx,
}
}
// 处理进入前逻辑w,r
// func (l *UploadCallbackLogic) BeforeLogic(w http.ResponseWriter, r *http.Request) {
// }
// 处理逻辑后 w,r 如:重定向, resp 必须重新处理
// func (l *UploadCallbackLogic) AfterLogic(w http.ResponseWriter, r *http.Request, resp *basic.Response) {
// // httpx.OkJsonCtx(r.Context(), w, resp)
// }
func (l *UploadCallbackLogic) UploadCallback(req *types.UploadCallbackReq, userinfo *auth.UserInfo) (resp *basic.Response) {
// 返回值必须调用Set重新返回, resp可以空指针调用 resp.SetStatus(basic.CodeOK, data)
// userinfo 传入值时, 一定不为null
return resp.SetStatus(basic.CodeOK)
}

View File

@ -0,0 +1,226 @@
package logic
import (
"fusenapi/model/gmodel"
"fusenapi/utils/auth"
"fusenapi/utils/basic"
"io"
"net/http"
"time"
"context"
"fusenapi/server/upload/internal/svc"
"fusenapi/server/upload/internal/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/mr"
)
type UploadFilesBackendLogic struct {
logx.Logger
ctx context.Context
svcCtx *svc.ServiceContext
r *http.Request
}
func NewUploadFilesBackendLogic(r *http.Request, svcCtx *svc.ServiceContext) *UploadFilesBackendLogic {
return &UploadFilesBackendLogic{
Logger: logx.WithContext(r.Context()),
ctx: r.Context(),
svcCtx: svcCtx,
r: r,
}
}
// 处理进入前逻辑w,r
// func (l *UploadFilesBackendLogic) BeforeLogic(w http.ResponseWriter, r *http.Request) {
// }
// 处理逻辑后 w,r 如:重定向, resp 必须重新处理
// func (l *UploadFilesBackendLogic) AfterLogic(w http.ResponseWriter, r *http.Request, resp *basic.Response) {
// // httpx.OkJsonCtx(r.Context(), w, resp)
// }
func (l *UploadFilesBackendLogic) UploadFilesBackend(req *types.UploadFilesReq, userinfo *auth.UserInfo) (resp *basic.Response) {
// 返回值必须调用Set重新返回, resp可以空指针调用 resp.SetStatus(basic.CodeOK, data)
// userinfo 传入值时, 一定不为null
if userinfo.IsOnlooker() {
// 如果是,返回未授权的错误码
return resp.SetStatus(basic.CodeUnAuth)
}
// 定义用户ID和S3键名格式
var userId int64
var guestId int64
// 检查用户是否是游客
if userinfo.IsGuest() {
// 如果是使用游客ID和游客键名格式
guestId = userinfo.GuestId
} else {
// 否则使用用户ID和用户键名格式
userId = userinfo.UserId
}
var aa = make([]types.UploadInfo, 2)
aa[0].FileKeys = "202308011632"
aa[1].FileKeys = "202308011633"
req.UploadInfo = aa
var fileLen = len(req.UploadInfo)
if fileLen == 0 {
return resp.SetStatus(basic.CodeFileUploadErr, "file upload err,no files")
}
if req.ApiType == 1 && fileLen > 100 {
return resp.SetStatus(basic.CodeFileUploadErr, "file upload err, files count is beyond the maximum")
}
//设置内存大小
l.r.ParseMultipartForm(32 << 20)
//获取上传的文件组
files := l.r.MultipartForm.File["file"]
// 设置AWS会话的区域
l.svcCtx.AwsSession.Config.Region = aws.String("us-west-1")
// 创建新的S3服务实例
svc := s3.New(l.svcCtx.AwsSession)
// 定义S3请求和当前时间
var s3req *request.Request
var uploadBucket = req.UploadBucket
resourceModel := gmodel.NewFsResourceModel(l.svcCtx.MysqlConn)
result, err := mr.MapReduce(func(source chan<- interface{}) {
for i, info := range req.UploadInfo {
fileType := files[i].Header.Get("Content-Type")
// 打开文件
file, err := files[i].Open()
if err != nil {
logx.Error(err)
}
defer file.Close()
// 读取数据流
ioData, err := io.ReadAll(file)
if err != nil {
logx.Error(err)
}
// 一系列业务逻辑....验证类型,文件大小
var fileKey string = info.FileKeys
source <- uploadData{
FileKey: fileKey,
FileType: fileType,
Metadata: info.Metadata,
FileData: ioData,
ApiType: req.ApiType,
}
}
}, func(item interface{}, writer mr.Writer[interface{}], cancel func(error)) {
var uploadUrl = uploadUrl{}
uploadDataInfo := item.(uploadData)
var resourceId string = uploadDataInfo.FileKey
// 查询数据库
resourceInfo, err := resourceModel.FindOneById(l.ctx, resourceId)
if err == nil && resourceInfo.ResourceId != "" {
uploadUrl.Status = 1
uploadUrl.ResourceId = resourceId
uploadUrl.Url = *resourceInfo.ResourceUrl
uploadUrl.Key = resourceId
} else {
// 创建S3对象存储请求
s3req, _ = svc.PutObjectRequest(
&s3.PutObjectInput{
Bucket: &uploadBucket,
Key: &uploadDataInfo.FileKey,
},
)
// 设置请求体为文件数据
s3req.SetBufferBody(uploadDataInfo.FileData)
// 发送请求
err = s3req.Send()
// 检查是否有错误
if err != nil {
logx.Error(err)
uploadUrl.Status = 0
uploadUrl.Url = ""
uploadUrl.Key = uploadDataInfo.FileKey
} else {
var url = s3req.HTTPRequest.URL.String()
// 打印请求URL
logx.Info(url)
uploadUrl.Status = 1
uploadUrl.Url = url
uploadUrl.Key = uploadDataInfo.FileKey
uploadUrl.ResourceId = resourceId
var version string = "0.0.1"
var nowTime = time.Now()
_, err = resourceModel.CreateOrUpdate(l.ctx, &gmodel.FsResource{
ResourceId: resourceId,
UserId: &userId,
GuestId: &guestId,
ResourceType: &uploadDataInfo.FileType,
ResourceUrl: &url,
Version: &version,
UploadedAt: &nowTime,
Metadata: &uploadDataInfo.Metadata,
ApiType: &uploadDataInfo.ApiType,
})
if err != nil {
logx.Error(err)
}
}
}
// Notice 这个必须加!
writer.Write(uploadUrl)
}, func(pipe <-chan interface{}, writer mr.Writer[interface{}], cancel func(error)) {
var uploadUrlList = make(map[string][]*uploadUrl)
var uploadUrlListFail []*uploadUrl
var uploadUrlListSuccess []*uploadUrl
for p := range pipe {
var uploadUrl = p.(uploadUrl)
if uploadUrl.Status == 1 {
uploadUrlListSuccess = append(uploadUrlListSuccess, &uploadUrl)
} else {
uploadUrlListFail = append(uploadUrlListFail, &uploadUrl)
}
}
// Notice 这个必须加!
uploadUrlList["success"] = uploadUrlListSuccess
uploadUrlList["fail"] = uploadUrlListFail
writer.Write(uploadUrlList)
})
if err != nil {
logx.Error(err)
}
// 返回成功的响应和上传URL
return resp.SetStatus(basic.CodeOK, map[string]interface{}{
"upload_urls": result,
})
}
type uploadData struct {
ApiType int64 `json:"api_type"`
FileType string `json:"file_type"`
FileKey string `json:"file_key"`
Metadata string `json:"metadata"`
FileData []byte `fsfile:"data"`
}
type uploadUrl struct {
Status int64 `json:"status"`
ResourceId string `json:"resource_id"`
Url string `json:"url"`
Key string `json:"key"`
Metadata string `json:"metadata"`
}

View File

@ -0,0 +1,45 @@
package logic
import (
"fusenapi/utils/auth"
"fusenapi/utils/basic"
"context"
"fusenapi/server/upload/internal/svc"
"fusenapi/server/upload/internal/types"
"github.com/zeromicro/go-zero/core/logx"
)
type UploadFilesFrontendLogic struct {
logx.Logger
ctx context.Context
svcCtx *svc.ServiceContext
}
func NewUploadFilesFrontendLogic(ctx context.Context, svcCtx *svc.ServiceContext) *UploadFilesFrontendLogic {
return &UploadFilesFrontendLogic{
Logger: logx.WithContext(ctx),
ctx: ctx,
svcCtx: svcCtx,
}
}
// 处理进入前逻辑w,r
// func (l *UploadFilesFrontendLogic) BeforeLogic(w http.ResponseWriter, r *http.Request) {
// }
// 处理逻辑后 w,r 如:重定向, resp 必须重新处理
// func (l *UploadFilesFrontendLogic) AfterLogic(w http.ResponseWriter, r *http.Request, resp *basic.Response) {
// // httpx.OkJsonCtx(r.Context(), w, resp)
// }
func (l *UploadFilesFrontendLogic) UploadFilesFrontend(req *types.UploadFilesReq, userinfo *auth.UserInfo) (resp *basic.Response) {
// 返回值必须调用Set重新返回, resp可以空指针调用 resp.SetStatus(basic.CodeOK, data)
// userinfo 传入值时, 一定不为null
// 检查用户是否是旁观者,旁观者没有文件上传权限
return resp.SetStatus(basic.CodeOK)
}

View File

@ -5,6 +5,25 @@ import (
"fusenapi/utils/basic"
)
type UploadInfo struct {
FileKeys string `form:"file_keys,optional"` // 上传唯一标识信息
Metadata string `form:"file_keys,optional"` // 上传文件额外信息
}
type UploadFilesReq struct {
ApiType int64 `form:"api_type,options=[1,2],default=1"` // 调用类型1=对外2=对内
UploadBucket string `form:"upload_bucket"` // 上传桶名
UploadInfo []UploadInfo `form:"upload_info,optional"` // 上传信息
}
type UploadCallbackReq struct {
FileType string `form:"file_type"` // 文件类型 / fbx / hdr
UploadKey string `form:"upload_key"` // 上传KEY
UploadBucket string `form:"upload_bucket"` // 上传桶名
Version string `form:"version,optional"` // 版本信息
Metadata string `form:"metadata,optional"` // 元数据,json格式,存储图像分率
}
type RequestUpFile struct {
UpFile string `form:"upfile"`
IsCut string `form:"is_cut"` // 是否裁剪

View File

@ -12,17 +12,49 @@ import "basic.api"
service upload {
@handler UploadUpFileHandler
get /api/upload/up-file(RequestUpFile) returns (response);
@handler UploadFileFrontendHandler
post /api/upload/upload-file-frontend(RequestUploadFileFrontend) returns (response);
@handler UploadFileBackendHandler
post /api/upload/upload-file-backend(RequestUploadFileBackend) returns (response);
//生成二维码
@handler UploadQrcodeHandler
post /api/upload/qrcode(UploadQrcodeReq) returns (response);
// 上传文件发起--单个文件--后端上传
@handler UploadFilesBackendHandler
post /api/upload/upload-files-backend(UploadFilesReq) returns (response);
// 上传文件发起--多个文件--前端上传
@handler UploadFilesFrontendHandler
post /api/upload/upload-files-frontend(UploadFilesReq) returns (response);
// 上传文件回调
@handler UploadCallbackHandler
post /api/upload/upload-callback(UploadCallbackReq) returns (response);
}
type (
UploadInfo {
FileKeys string `form:"file_keys,optional"` // 上传唯一标识信息
Metadata string `form:"file_keys,optional"` // 上传文件额外信息
}
UploadFilesReq {
ApiType int64 `form:"api_type,options=[1,2],default=1"` // 调用类型1=对外2=对内
UploadBucket string `form:"upload_bucket"` // 上传桶名
UploadInfo []UploadInfo `form:"upload_info,optional"` // 上传信息
}
UploadCallbackReq {
FileType string `form:"file_type"` // 文件类型 / fbx / hdr
UploadKey string `form:"upload_key"` // 上传KEY
UploadBucket string `form:"upload_bucket"` // 上传桶名
Version string `form:"version,optional"` // 版本信息
Metadata string `form:"metadata,optional"` // 元数据,json格式,存储图像分率
}
)
type RequestUpFile {
UpFile string `form:"upfile"`
IsCut string `form:"is_cut"` // 是否裁剪

View File

@ -76,6 +76,8 @@ var (
CodeAesCbcEncryptionErr = &StatusResponse{5106, "encryption data err"} // 加密数据失败
CodeAesCbcDecryptionErr = &StatusResponse{5107, "decryption data err"} // 解密数据失败
CodeFileUploadErr = &StatusResponse{5110, "file upload err"} // 文件上传失败
)
type Response struct {

View File

@ -107,3 +107,5 @@ func FormatS3KeyNameGuest(guestid int64, now time.Time, env string, category Typ
}
}
// generateS3KeyName 生成