Merge branch 'dataset-urchin' of https://openi.pcl.ac.cn/OpenI/aiforge into dataset-urchin

models/ai_model_manage.go

@@ -1158,7 +1158,7 @@ func GetAimodelByID(id string) (*AiModelManage, error) {
 	return aimodel, nil
 }
 
-func DeleteAimodel(doer *User, uid int64, aimodelId string) error {
+func DeleteAimodel(ctx DBContext, doer *User, uid int64, aimodelId string) error {
 	// In case is a organization.
 	org, err := GetUserByID(uid)
 	if err != nil {
@@ -1170,7 +1170,7 @@ func DeleteAimodel(doer *User, uid int64, aimodelId string) error {
 		}
 	}
 
-	sess := x
+	sess := ctx.e
 	aimodel := &AiModelManage{ID: aimodelId, OwnerID: uid}
 	has, err := sess.Get(aimodel)
 	if err != nil {

models/error.go

@@ -2163,3 +2163,15 @@ func IsErrCannotStopSavingImageJob(err error) bool {
 func (err ErrCannotStopSavingImageJob) Error() string {
 	return fmt.Sprintf("job is saving image, can not be stopped")
 }
+
+type ErrObjectStatusInvalid struct {
+}
+
+func IsErrObjectStatusInvalid(err error) bool {
+	_, ok := err.(ErrObjectStatusInvalid)
+	return ok
+}
+
+func (err ErrObjectStatusInvalid) Error() string {
+	return fmt.Sprintf("dataset.resource_in_use")
+}

models/hf_model.go

@@ -1,11 +1,12 @@
 package models
 
 import (
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/timeutil"
 	"fmt"
 	"strings"
 	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/timeutil"
 	"xorm.io/builder"
 )
 
@@ -349,6 +350,17 @@ func DeleteHfFilesByModelId(id string) error {
 	return nil
 }
 
+func DeleteHfFilesByModelIdWithContext(ctx DBContext, id string) error {
+	sess := ctx.e
+	_, err := sess.Delete(&HfModelFile{
+		ModelId: id,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func GetHfModelSizeByModelId(modelId string) (int64, error) {
 	sess := x.NewSession()
 	defer sess.Close()

modules/urchin_v2/client.go

@@ -24,16 +24,6 @@ type UrchinV2Client struct {
 	ObsClient    *obs.ObsClient
 }
 
-const UrchinSuccessCode = 0
-
-func IsUrchinRespSuccess(resp *urchin_module.BaseResp) bool {
-	return resp.Code == UrchinSuccessCode
-}
-
-func UrchinRespToError(resp *urchin_module.BaseResp) error {
-	return fmt.Errorf("urchin operate error.[%d]%s", resp.Code, resp.Message)
-}
-
 func NewUrchinClient(address string,
 	reqTimeout int64,
 	maxConnection int) *UrchinV2Client {
@@ -70,8 +60,8 @@ func (client *UrchinV2Client) CreateCollection(name string) (string, error) {
 	if nil != err {
 		return "", err
 	}
-	if createObjectResp.Code != 0 || createObjectResp.ObjUuid == "" {
-		return "", fmt.Errorf("urchin create object error.resp = %+v", createObjectResp)
+	if !createObjectResp.IsSuccess() {
+		return "", createObjectResp.ToError()
 	}
 	return createObjectResp.ObjUuid, nil
 }
@@ -89,8 +79,8 @@ func (client *UrchinV2Client) CreatePutObjectSignedUrl(objUuid string, relativeP
 	if nil != err {
 		return "", err
 	}
-	if resp.Code != 0 {
-		return "", fmt.Errorf("urchin CreatePutObjectSignedUrl error.[%d](%s)] resp=%v", resp.Code, resp.Message, resp)
+	if !resp.IsSuccess() {
+		return "", resp.ToError()
 	}
 
 	return resp.SignedUrl, nil
@@ -265,7 +255,9 @@ func (client *UrchinV2Client) GetObject(objuuid, path string) (*obs.GetObjectOut
 	if err != nil {
 		return nil, err
 	}
-
+	if !objectResp.IsSuccess() {
+		return nil, objectResp.ToError()
+	}
 	output, err := client.ObsClient.GetObjectWithSignedUrl(
 		objectResp.SignedUrl,
 		http.Header{})
@@ -289,7 +281,9 @@ func (client *UrchinV2Client) GetObjectMeta(objuuid, path string) (*obs.GetObjec
 	if err != nil {
 		return nil, err
 	}
-
+	if !objectResp.IsSuccess() {
+		return nil, objectResp.ToError()
+	}
 	output, err := client.ObsClient.GetObjectMetadataWithSignedUrl(
 		objectResp.SignedUrl,
 		http.Header{})
@@ -315,8 +309,8 @@ func (client *UrchinV2Client) FinishSuccessTask(taskId int32) error {
 	if err != nil {
 		return err
 	}
-	if !IsUrchinRespSuccess(resp) {
-		return UrchinRespToError(resp)
+	if !resp.IsSuccess() {
+		return resp.ToError()
 	}
 	return nil
 }
@@ -333,8 +327,8 @@ func (client *UrchinV2Client) FinishFailedTask(taskId int32) error {
 	if err != nil {
 		return err
 	}
-	if !IsUrchinRespSuccess(resp) {
-		return UrchinRespToError(resp)
+	if !resp.IsSuccess() {
+		return resp.ToError()
 	}
 	return nil
 }
@@ -350,6 +344,9 @@ func (client *UrchinV2Client) GetObjectSignedUrl(objuuid, path string) (string,
 	if err != nil {
 		return "", err
 	}
+	if !objectResp.IsSuccess() {
+		return "", objectResp.ToError()
+	}
 	return objectResp.SignedUrl, nil
 }
 
@@ -388,8 +385,8 @@ func (client *UrchinV2Client) DeleteFile(objuuid, sourcePath string) error {
 	if err != nil {
 		return err
 	}
-	if !IsUrchinRespSuccess(resp) {
-		return UrchinRespToError(resp)
+	if !resp.IsSuccess() {
+		return resp.ToError()
 	}
 
 	return nil
@@ -405,8 +402,8 @@ func (client *UrchinV2Client) DeleteObject(objuuid string) error {
 	if err != nil {
 		return err
 	}
-	if !IsUrchinRespSuccess(resp) {
-		return UrchinRespToError(resp)
+	if !resp.IsSuccess() {
+		return resp.ToError()
 	}
 
 	return nil
@@ -430,6 +427,9 @@ func (client *UrchinV2Client) InitiateMultipartUpload(objuuid, path string) (str
 	if err != nil {
 		return "", err
 	}
+	if !objectResp.IsSuccess() {
+		return "", objectResp.ToError()
+	}
 	output, err := client.ObsClient.InitiateMultipartUploadWithSignedUrl(
 		objectResp.SignedUrl,
 		http.Header{})
@@ -453,6 +453,9 @@ func (client *UrchinV2Client) CreateMultipartUploadSignedUrl(objuuid, uploadId,
 	if err != nil {
 		return "", err
 	}
+	if !output.IsSuccess() {
+		return "", output.ToError()
+	}
 	return output.SignedUrl, nil
 }
 
@@ -467,6 +470,9 @@ func (client *UrchinV2Client) CompleteMultiPartUpload(objuuid, uploadId, source
 	if err != nil {
 		return err
 	}
+	if !urlOutput.IsSuccess() {
+		return urlOutput.ToError()
+	}
 
 	allParts, err := client.ListAllParts(objuuid, uploadId, source)
 	if err != nil {
@@ -529,6 +535,9 @@ func (client *UrchinV2Client) ListAllParts(objuuid, uploadID, path string) (outp
 		if err != nil {
 			return nil, err
 		}
+		if !objectResp.IsSuccess() {
+			return nil, objectResp.ToError()
+		}
 		temp, err := client.ObsClient.ListPartsWithSignedUrl(
 			objectResp.SignedUrl,
 			http.Header{})

modules/urchin_v2/module/urchin.go

@@ -2,7 +2,10 @@ package module
 
 import (
 	"encoding/xml"
+	"fmt"
 	"time"
+
+	"code.gitea.io/gitea/models"
 )
 
 const (
@@ -82,12 +85,20 @@ const (
 
 	TaskFResultESuccess = 1
 	TaskFResultEFailed  = 2
+
+	UrchinSuccessCode       = 0
+	ObjectStatusInvalidCode = 10350
 )
 
 var TaskTypeOnlyServiceRetry = map[int32]bool{
 	TaskTypeMigrate: true,
 	TaskTypeCopy:    true}
 
+type UrchinError interface {
+	ToError() error
+	IsSuccess() bool
+}
+
 type BaseResp struct {
 	// @inject_tag: json:"code"
 	Code int32 `protobuf:"varint,1,opt,name=code,proto3" json:"code"`
@@ -97,6 +108,17 @@ type BaseResp struct {
 	RequestId string `protobuf:"bytes,3,opt,name=request_id,proto3" json:"request_id"`
 }
 
+func (r *BaseResp) ToError() error {
+	if r.Code == ObjectStatusInvalidCode {
+		return models.ErrObjectStatusInvalid{}
+	}
+	return fmt.Errorf("urchin operate error.[%d]%s", r.Code, r.Message)
+}
+
+func (r *BaseResp) IsSuccess() bool {
+	return r.Code == UrchinSuccessCode
+}
+
 type CreateInitiateMultipartUploadSignedUrlReq struct {
 	// @inject_tag: json:"task_id"
 	TaskId int32 `protobuf:"varint,1,opt,name=task_id,proto3" json:"task_id"`
@@ -265,6 +287,17 @@ type CreateSignedUrlResp struct {
 	Header map[string]*HeaderValues `protobuf:"bytes,5,rep,name=header,proto3" json:"header" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
+func (r *CreateSignedUrlResp) ToError() error {
+	if r.Code == ObjectStatusInvalidCode {
+		return models.ErrObjectStatusInvalid{}
+	}
+	return fmt.Errorf("urchin operate error.[%d]%s", r.Code, r.Message)
+}
+
+func (r *CreateSignedUrlResp) IsSuccess() bool {
+	return r.Code == UrchinSuccessCode
+}
+
 type HeaderValues struct {
 	// @inject_tag: json:"values"
 	Values []string `protobuf:"bytes,1,rep,name=values,proto3" json:"values"`
@@ -362,6 +395,17 @@ type CreateObjectResp struct {
 	DeploymentInfo *ObjDeploymentData `protobuf:"bytes,5,opt,name=deployment_info,proto3" json:"deployment_info"`
 }
 
+func (r *CreateObjectResp) ToError() error {
+	if r.Code == ObjectStatusInvalidCode {
+		return models.ErrObjectStatusInvalid{}
+	}
+	return fmt.Errorf("urchin operate error.[%d]%s", r.Code, r.Message)
+}
+
+func (r *CreateObjectResp) IsSuccess() bool {
+	return r.Code == UrchinSuccessCode
+}
+
 type UploadObjectReq struct {
 	// @inject_tag: json:"user_id"
 	UserId string `protobuf:"bytes,1,opt,name=user_id,proto3" json:"user_id"`
@@ -409,6 +453,17 @@ type UploadObjectResp struct {
 	ObjUuid string `protobuf:"bytes,7,opt,name=obj_uuid,proto3" json:"obj_uuid"`
 }
 
+func (r *UploadObjectResp) ToError() error {
+	if r.Code == ObjectStatusInvalidCode {
+		return models.ErrObjectStatusInvalid{}
+	}
+	return fmt.Errorf("urchin operate error.[%d]%s", r.Code, r.Message)
+}
+
+func (r *UploadObjectResp) IsSuccess() bool {
+	return r.Code == UrchinSuccessCode
+}
+
 type UploadFileReq struct {
 	// @inject_tag: json:"user_id"
 	UserId string `protobuf:"bytes,1,opt,name=user_id,proto3" json:"user_id"`
@@ -452,6 +507,17 @@ type UploadFileResp struct {
 	NodeName string `protobuf:"bytes,6,opt,name=node_name,proto3" json:"node_name"`
 }
 
+func (r *UploadFileResp) ToError() error {
+	if r.Code == ObjectStatusInvalidCode {
+		return models.ErrObjectStatusInvalid{}
+	}
+	return fmt.Errorf("urchin operate error.[%d]%s", r.Code, r.Message)
+}
+
+func (r *UploadFileResp) IsSuccess() bool {
+	return r.Code == UrchinSuccessCode
+}
+
 type DownloadFileReq struct {
 	// @inject_tag: json:"user_id"
 	UserId string `protobuf:"bytes,1,opt,name=user_id,proto3" json:"user_id"`

options/locale/locale_en-US.ini

@@ -976,6 +976,7 @@ description = Description
 description_format_err=Description's length can be up to %s characters long.
 dataset_name_exist = Dataset English name already exists
 dataset_alias_exist = Dataset Chinese name already exists
+resource_in_use = Resource is in use, operation not possible, please try again later.
 over_preview_size = The file size exceeds the preview limit
 unsupported_preview_file_type = The file type does not support preview
 create_dataset = Create Dataset
@@ -3790,4 +3791,4 @@ change_team_access_not_allowed = Changing team access for subject has been restr
 org_not_allowed_to_upload_dataset_file = Datasets owned by organizations currently do not allow file uploads. If this feature is required, please wait for future updates.
 
 [aimodel]
-org_not_allowed_to_upload_aimodel_file = Aimodels owned by organizations currently do not allow file uploads. If this feature is required, please wait for future updates.
+org_not_allowed_to_upload_aimodel_file = Aimodels owned by organizations currently do not allow file uploads. If this feature is required, please wait for future updates.

options/locale/locale_zh-CN.ini

@@ -985,6 +985,7 @@ description=描述
 description_format_err=描述最多允许输入%s个字符。
 dataset_name_exist = 数据集英文名称已存在。
 dataset_alias_exist = 数据集中文名称已存在。
+resource_in_use = 资源在使用中，无法执行此操作，请稍后重试
 over_preview_size = 该文件大小超过了预览限制。
 unsupported_preview_file_type = 该文件类型不支持预览。
 create_dataset=创建数据集

routers/api/v1/aimodel/aimodel.go

@@ -2,6 +2,15 @@ package aimodel
 
 import (
 	"archive/zip"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+
 	"code.gitea.io/gitea/entity"
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
@@ -14,14 +23,6 @@ import (
 	"code.gitea.io/gitea/services/ai_task_service/storage_helper"
 	"code.gitea.io/gitea/services/dynconfig/fetcher"
 	"code.gitea.io/gitea/services/subject_service"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"path"
-	"strconv"
-	"strings"
 )
 
 func CreateAimodel(ctx *context.Context, req entity.CreateAimodelReq) {
@@ -183,7 +184,7 @@ func DelAimodel(ctx *context.Context) {
 	aimodel := ctx.AccessContext.Aimodel
 	err := subject_service.DeleteAimodel(ctx.User, aimodel)
 	if err != nil {
-		ctx.JSON(http.StatusOK, response.ResponseError(err))
+		ctx.JSON(http.StatusOK, response.OuterTrBizError(response.NewBizError(err), ctx.Locale))
 		return
 	}
 	ctx.JSON(http.StatusOK, response.OuterSuccess())
@@ -477,7 +478,7 @@ func DeleteAimodelFile(ctx *context.Context) {
 	err := subject_service.DeleteAimodelFile(aimodel, parentDir, fileName)
 	if err != nil {
 		log.Error("DeleteAimodelFile failed, aimodel.ID=%s parentDir=%s fileName=%s err=%v", aimodel.ID, parentDir, fileName, err)
-		ctx.JSON(http.StatusOK, response.ResponseError(err))
+		ctx.JSON(http.StatusOK, response.OuterTrBizError(response.NewBizError(err), ctx.Locale))
 		return
 	}
 	ctx.JSON(http.StatusOK, response.OuterSuccess())

routers/api/v1/dataset/dataset.go

@@ -710,7 +710,7 @@ func DeleteDatasetFile(ctx *context.Context) {
 	err := subject_service.DeleteDatasetFile(dataset, parentDir, fileName)
 	if err != nil {
 		log.Error("DeleteFile failed, dataset.ID=%s parentDir=%s fileName=%s err=%v", dataset.ID, parentDir, fileName, err)
-		ctx.JSON(http.StatusOK, response.ResponseError(err))
+		ctx.JSON(http.StatusOK, response.OuterTrBizError(response.NewBizError(err), ctx.Locale))
 		return
 	}
 	ctx.JSON(http.StatusOK, response.OuterSuccess())
@@ -1136,7 +1136,7 @@ func DelDataset(ctx *context.Context) {
 	dataset := ctx.AccessContext.Dataset
 	err := subject_service.DeleteDataset(ctx.User, dataset)
 	if err != nil {
-		ctx.JSON(http.StatusOK, response.ResponseError(err))
+		ctx.JSON(http.StatusOK, response.OuterTrBizError(response.NewBizError(err), ctx.Locale))
 		return
 	}
 	ctx.JSON(http.StatusOK, response.OuterSuccess())

routers/response/response_list.go

@@ -53,8 +53,8 @@ var PORT_RANGE = &BizError{Code: 2031, DefaultMsg: "The port range is 8000 to 88
 var ENDPOINT_NOT_START_SLASH = &BizError{Code: 2032, DefaultMsg: "Custom paths need to comply with URL specifications", TrCode: "ai_task.endpoint_not_start_slash"}
 var ENDPOINT_MUST_BE_VALID = &BizError{Code: 2033, DefaultMsg: "Custom paths can only include strings, numbers, and /", TrCode: "ai_task.endpoint_must_be_valid"}
 var CAN_NOT_STOP_SAVING_IMAGE_JOB = &BizError{Code: 2034, DefaultMsg: "Cannot stop the AI task while it is saving the image.", TrCode: "ai_task.can_not_stop_saving_image_job"}
-var CAN_NOT_FINETUNE_EXPERIENCE=&BizError{Code: 2035, DefaultMsg: "The task can not be exepirenced online.", TrCode: "ai_task.can_not_finetune_experience"}
-var CAN_NOT_Eval=&BizError{Code: 2036, DefaultMsg: "The task can not be evaluated online.", TrCode: "ai_task.can_not_eval"}
+var CAN_NOT_FINETUNE_EXPERIENCE = &BizError{Code: 2035, DefaultMsg: "The task can not be exepirenced online.", TrCode: "ai_task.can_not_finetune_experience"}
+var CAN_NOT_Eval = &BizError{Code: 2036, DefaultMsg: "The task can not be evaluated online.", TrCode: "ai_task.can_not_eval"}
 
 // 登录相关错误
 var QR_CODE_EXPIRED = &BizError{Code: 3001, DefaultMsg: "It has expired, please scan the QR code again", TrCode: "form.qr_code_expire"}
@@ -71,6 +71,7 @@ var DATASET_EXIST = &BizError{Code: 4007, DefaultMsg: "dataset exists", TrCode:
 var DATASET_PATH_EMPTY = &BizError{Code: 4008, DefaultMsg: "dataset path empty", TrCode: ""}
 var DATASET_ALIAS_INVALID = &BizError{Code: 4009, DefaultMsg: "Dataset alias is invalid", TrCode: "dataset.alias_format_err"}
 var DATASET_ALIAS_EXIST = &BizError{Code: 4010, DefaultMsg: "Dataset alias already exists", TrCode: "dataset.dataset_alias_exist"}
+var RESOURCE_IN_USE = &BizError{Code: 4011, DefaultMsg: "Resource is in use, operation not possible, please try again later.", TrCode: "dataset.resource_in_use"}
 
 // 权限相关错误
 var ORG_NOT_ALLOWED_TO_BE_COLLABORATOR = &BizError{Code: 5001, DefaultMsg: "Organizations cannot be added as a collaborator", TrCode: "access.org_not_allowed_to_be_collaborator"}

services/subject_service/aimodel_service.go

@@ -154,21 +154,28 @@ func DeleteAimodel(doer *models.User, aimodel *models.AiModelManage) error {
 		return err
 	}
 
-	err = models.DeleteAimodel(doer, aimodel.OwnerID, aimodel.ID)
-	if err != nil {
-		log.Error("DeleteAimodel err,dataset=%+v,err=%v", aimodel, err)
-	}
-	go DeleteAimodelStorage(aimodel)
-	notification.NotifyDeleteAimodel(doer, owner, aimodel)
-
-	// delete hf transfer file record db
-	if aimodel.ModelType == models.MODEL_HF_TYPE {
-		err := models.DeleteHfFilesByModelId(aimodel.ID)
+	err = models.WithTx(func(ctx models.DBContext) error {
+		err := models.DeleteAimodel(ctx, doer, aimodel.OwnerID, aimodel.ID)
 		if err != nil {
-			log.Info("DeleteHfFilesByModelId error." + err.Error())
+			log.Error("DeleteAimodel err,dataset=%+v,err=%v", aimodel, err)
 			return err
 		}
-	}
+		err = DeleteAimodelStorage(aimodel)
+		if err != nil {
+			log.Error("DeleteAimodel DeleteAimodelStorage err,aimodel=%+v,err=%v", aimodel, err)
+			return err
+		}
+		// delete hf transfer file record db
+		if aimodel.ModelType == models.MODEL_HF_TYPE {
+			err := models.DeleteHfFilesByModelIdWithContext(ctx, aimodel.ID)
+			if err != nil {
+				log.Info("DeleteHfFilesByModelId error." + err.Error())
+				return err
+			}
+		}
+		notification.NotifyDeleteAimodel(doer, owner, aimodel)
+		return nil
+	})
 
 	return nil
 }
@@ -180,6 +187,7 @@ func DeleteAimodelStorage(aimodel *models.AiModelManage) error {
 			log.Error("PANIC:%v", combinedErr)
 		}
 	}()
+
 	helper := storage_helper.SelectStorageHelperFromStorageType(entity.StorageType(aimodel.StorageType))
 	if helper == nil {
 		log.Error("GetUploadHelper failed, dataType=%d", models.AimodelSubject)
@@ -189,7 +197,7 @@ func DeleteAimodelStorage(aimodel *models.AiModelManage) error {
 	if err == nil && remainingFiles {
 		tmpErr := models.InsertStorageDeleteFailedAimodel(aimodel)
 		if tmpErr != nil {
-			log.Error("InsertStorageDeleteFailedDataset failed, datasetId=%s, datasetName=%s err=%v", aimodel.ID, aimodel.Name, tmpErr)
+			log.Error("InsertStorageDeleteFailedAimodel failed, aimodelId=%s, datasetName=%s err=%v", aimodel.ID, aimodel.Name, tmpErr)
 		}
 	}
 	return err

services/subject_service/upload_service.go

@@ -152,13 +152,6 @@ func NewUploadMultipart(req entity.NewMultipartRequest) (*entity.NewMuiltipartRe
 
 	isLimited := storage_limit.IsSubjectFileUploadOverLimit(req.Size, req.SubjectAccessContext)
 	if isLimited {
-		if req.SubjectAccessContext.Owner.IsOrganization() {
-			if req.SubjectAccessContext.SubjectType == models.DatasetSubject {
-				return nil, errors.New("dataset_registry.org_not_allowed_to_upload_dataset_file")
-			} else {
-				return nil, errors.New("aimodel.org_not_allowed_to_upload_aimodel_file")
-			}
-		}
 		return nil, errors.New("common_error.file_size_storage_limit")
 	}