natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DSIP-45] Polish the Storage SPI (#16141)

This commit is contained in:
Wenjun Ruan 2024-06-17 15:24:56 +08:00 committed by GitHub
parent 61a6bd0b30
commit edeb1b2458
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
139 changed files with 6496 additions and 6973 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

517
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
View File

@ -28,42 +28,50 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_BY_T
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_PAGING;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UDF_FUNCTION_LIST_PAGING_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_FILE_IS_EMPTY;
import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_NOT_EXIST;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_RESOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_UDF_FUNCTION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_UDF_FUNCTION_NAME_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VIEW_RESOURCE_FILE_ON_LINE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VIEW_UDF_FUNCTION_ERROR;
import org.apache.dolphinscheduler.api.audit.OperatorLog;
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest;
import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.api.vo.ResourceItemVO;
import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import java.util.List;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@ -77,15 +85,14 @@ import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.google.common.io.Files;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.Parameters;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
/**
* resources controller
*/
@Tag(name = "RESOURCES_TAG")
@RestController
@RequestMapping("resources")
@ -94,41 +101,34 @@ public class ResourcesController extends BaseController {
@Autowired
private ResourcesService resourceService;
@Autowired
private UdfFuncService udfFuncService;
/**
* @param loginUser login user
* @param type type
* @param alias alias
* @param pid parent id
* @param currentDir current directory
* @return create result code
*/
@Operation(summary = "createDirectory", description = "CREATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")),
@Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class))})
@PostMapping(value = "/directory")
@ApiException(CREATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FOLDER_CREATE)
public Result<Object> createDirectory(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
public Result<Void> createDirectory(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "pid") int pid,
@RequestParam(value = "currentDir") String currentDir) {
// todo verify the directory name
return resourceService.createDirectory(loginUser, alias, type, pid, currentDir);
@RequestParam(value = "name") String directoryName,
@RequestParam(value = "currentDir") String parentDirectory) {
CreateDirectoryRequest createDirectoryRequest = CreateDirectoryRequest.builder()
.loginUser(loginUser)
.directoryName(directoryName)
.type(type)
.parentAbsoluteDirectory(parentDirectory)
.build();
resourceService.createDirectory(createDirectoryRequest);
return Result.success(null);
}
/**
* create resource
*
* @return create result code
*/
@Operation(summary = "createResource", description = "CREATE_RESOURCE_NOTES")
@Operation(summary = "uploadFile", description = "CREATE_FILE")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@ -137,224 +137,24 @@ public class ResourcesController extends BaseController {
@PostMapping()
@ApiException(CREATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_CREATE)
public Result<Object> createResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
public Result<Void> createFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "name") String fileName,
@RequestParam("file") MultipartFile file,
@RequestParam(value = "currentDir") String currentDir) {
// todo verify the file name
return resourceService.uploadResource(loginUser, alias, type, file, currentDir);
@RequestParam(value = "currentDir") String parentDirectoryAbsolutePath) {
CreateFileRequest uploadFileRequest = CreateFileRequest.builder()
.loginUser(loginUser)
.fileName(fileName)
.file(file)
.type(type)
.parentAbsoluteDirectory(parentDirectoryAbsolutePath)
.build();
resourceService.createFile(uploadFileRequest);
return Result.success();
}
/**
* update resource
*
* @param loginUser login user
* @param alias alias
* @param type resource type
* @param file resource file
* @return update result code
*/
@Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class))})
@PutMapping()
@ApiException(UPDATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_UPDATE)
public Result<Object> updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode", required = false) String tenantCode,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "file", required = false) MultipartFile file) {
return resourceService.updateResource(loginUser, fullName, tenantCode, alias, type, file);
}
/**
* query resources list
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
@Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<Object> queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "fullName") String fullName) {
Map<String, Object> result = resourceService.queryResourceList(loginUser, type, fullName);
return returnDataList(result);
}
/**
* query resources list paging
*
* @param loginUser login user
* @param type resource type
* @param searchVal search value
* @param pageNo page number
* @param pageSize page size
* @return resource list page
*/
@Operation(summary = "queryResourceListPaging", description = "QUERY_RESOURCE_LIST_PAGING_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")),
@Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)),
@Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")),
@Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20"))})
@GetMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_PAGING)
public Result<PageInfo<StorageEntity>> queryResourceListPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "type") ResourceType type,
@RequestParam("pageNo") Integer pageNo,
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam("pageSize") Integer pageSize) {
checkPageParams(pageNo, pageSize);
searchVal = ParameterUtils.handleEscapes(searchVal);
return resourceService.queryResourceListPaging(loginUser, fullName, tenantCode, type, searchVal, pageNo,
pageSize);
}
/**
* delete resource
*
* @param loginUser login user
* @return delete result code
*/
@Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))})
@DeleteMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_DELETE)
public Result<Object> deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode", required = false) String tenantCode) throws Exception {
return resourceService.delete(loginUser, fullName, tenantCode);
}
/**
* delete DATA_TRANSFER data
*
* @param loginUser login user
* @return delete result code
*/
@Operation(summary = "deleteDataTransferData", description = "Delete the N days ago data of DATA_TRANSFER ")
@Parameters({
@Parameter(name = "days", description = "N days ago", required = true, schema = @Schema(implementation = Integer.class))})
@DeleteMapping(value = "/data-transfer")
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
public DeleteDataTransferResponse deleteDataTransferData(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "days") Integer days) {
return resourceService.deleteDataTransferData(loginUser, days);
}
/**
* verify resource by alias and type
*
* @param loginUser login user
* @param fullName resource full name
* @param type resource type
* @return true if the resource name not exists, otherwise return false
*/
@Operation(summary = "verifyResourceName", description = "VERIFY_RESOURCE_NAME_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class))})
@GetMapping(value = "/verify-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR)
public Result<Object> verifyResourceName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "type") ResourceType type) {
return resourceService.verifyResourceName(fullName, type, loginUser);
}
/**
* query resources by type
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
@Operation(summary = "queryResourceByProgramType", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class))})
@GetMapping(value = "/query-by-type")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<Object> queryResourceJarList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "programType", required = false) ProgramType programType) {
return resourceService.queryResourceByProgramType(loginUser, type, programType);
}
/**
* query resource by file name and type
*
* @param loginUser login user
* @param fileName resource full name
* @param tenantCode tenantCode of the owner of the resource
* @param type resource type
* @return true if the resource name not exists, otherwise return false
*/
@Operation(summary = "queryResourceByFileName", description = "QUERY_BY_RESOURCE_FILE_NAME")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fileName", description = "RESOURCE_FILE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),})
@GetMapping(value = "/query-file-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
public Result<Object> queryResourceByFileName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fileName", required = false) String fileName,
@RequestParam(value = "tenantCode", required = false) String tenantCode,
@RequestParam(value = "type") ResourceType type) {
return resourceService.queryResourceByFileName(loginUser, fileName, type, tenantCode);
}
/**
* view resource file online
*
* @param loginUser login user
* @param skipLineNum skip line number
* @param limit limit
* @return resource content
*/
@Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100"))})
@GetMapping(value = "/view")
@ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR)
public Result viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "skipLineNum") int skipLineNum,
@RequestParam(value = "limit") int limit,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode) {
return resourceService.readResource(loginUser, fullName, tenantCode, skipLineNum, limit);
}
@Operation(summary = "onlineCreateResource", description = "ONLINE_CREATE_RESOURCE_NOTES")
@Operation(summary = "createFileFromContent", description = "ONLINE_CREATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fileName", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@ -365,27 +165,24 @@ public class ResourcesController extends BaseController {
@PostMapping(value = "/online-create")
@ApiException(CREATE_RESOURCE_FILE_ON_LINE_ERROR)
@OperatorLog(auditType = AuditType.FILE_CREATE)
public Result createResourceFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
public Result<Void> createFileFromContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "fileName") String fileName,
@RequestParam(value = "suffix") String fileSuffix,
@RequestParam(value = "content") String content,
@RequestParam(value = "currentDir") String currentDir) {
if (StringUtils.isEmpty(content)) {
log.error("resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
return resourceService.createResourceFile(loginUser, type, fileName, fileSuffix, content, currentDir);
@RequestParam(value = "content") String fileContent,
@RequestParam(value = "currentDir") String fileParentDirectoryAbsolutePath) {
CreateFileFromContentRequest createFileFromContentRequest = CreateFileFromContentRequest.builder()
.loginUser(loginUser)
.fileName(fileName + "." + fileSuffix)
.fileContent(fileContent)
.type(type)
.parentAbsoluteDirectory(fileParentDirectoryAbsolutePath)
.build();
resourceService.createFileFromContent(createFileFromContentRequest);
return Result.success();
}
/**
* edit resource file online
*
* @param loginUser login user
* @param content content
* @return update result code
*/
@Operation(summary = "updateResourceContent", description = "UPDATE_RESOURCE_NOTES")
@Operation(summary = "updateFileContent", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "fullName", description = "FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
@ -393,52 +190,170 @@ public class ResourcesController extends BaseController {
@PutMapping(value = "/update-content")
@ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR)
@OperatorLog(auditType = AuditType.FILE_UPDATE)
public Result updateResourceContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "content") String content) {
if (StringUtils.isEmpty(content)) {
log.error("The resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
return resourceService.updateResourceContent(loginUser, fullName, tenantCode, content);
public Result<Void> updateFileContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fileAbsolutePath,
@RequestParam(value = "content") String fileContent) {
UpdateFileFromContentRequest updateFileContentRequest = UpdateFileFromContentRequest.builder()
.loginUser(loginUser)
.fileContent(fileContent)
.fileAbsolutePath(fileAbsolutePath)
.build();
resourceService.updateFileFromContent(updateFileContentRequest);
return Result.success();
}
@Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class))})
@PutMapping()
@ApiException(UPDATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_UPDATE)
public Result<Void> updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsolutePath,
@RequestParam(value = "name") String resourceName,
@RequestParam(value = "file", required = false) MultipartFile file) {
if (StringUtils.isEmpty(Files.getFileExtension(resourceName))) {
RenameDirectoryRequest renameDirectoryRequest = RenameDirectoryRequest.builder()
.loginUser(loginUser)
.directoryAbsolutePath(resourceAbsolutePath)
.newDirectoryName(resourceName)
.build();
resourceService.renameDirectory(renameDirectoryRequest);
return Result.success();
}
if (file == null) {
RenameFileRequest renameFileRequest = RenameFileRequest.builder()
.loginUser(loginUser)
.fileAbsolutePath(resourceAbsolutePath)
.newFileName(resourceName)
.build();
resourceService.renameFile(renameFileRequest);
return Result.success();
}
UpdateFileRequest updateFileRequest = UpdateFileRequest.builder()
.loginUser(loginUser)
.fileAbsolutePath(resourceAbsolutePath)
.file(file)
.build();
resourceService.updateFile(updateFileRequest);
return Result.success();
}
@Operation(summary = "pagingResourceItemRequest", description = "PAGING_RESOURCE_ITEM_LIST")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")),
@Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)),
@Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")),
@Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20"))})
@GetMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_PAGING)
public Result<PageInfo<ResourceItemVO>> pagingResourceItemRequest(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsolutePath,
@RequestParam(value = "type") ResourceType resourceType,
@RequestParam(value = "searchVal", required = false) String resourceNameKeyWord,
@RequestParam("pageNo") Integer pageNo,
@RequestParam("pageSize") Integer pageSize) {
PagingResourceItemRequest pagingResourceItemRequest = PagingResourceItemRequest.builder()
.loginUser(loginUser)
.resourceAbsolutePath(resourceAbsolutePath)
.resourceType(resourceType)
.resourceNameKeyWord(StringUtils.trim(ParameterUtils.handleEscapes(resourceNameKeyWord)))
.pageNo(pageNo)
.pageSize(pageSize)
.build();
pagingResourceItemRequest.checkPageNoAndPageSize();
return Result.success(resourceService.pagingResourceItem(pagingResourceItemRequest));
}
// todo: this api is used for udf, we should remove it
@Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<List<ResourceComponent>> queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type) {
return Result.success(resourceService.queryResourceFiles(loginUser, type));
}
@Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "file:////tmp/dolphinscheduler/storage/default/resources/demo.sql"))
})
@DeleteMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_DELETE)
public Result<Void> deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsolutePath) {
DeleteResourceRequest deleteResourceRequest = DeleteResourceRequest.builder()
.loginUser(loginUser)
.resourceAbsolutePath(resourceAbsolutePath)
.build();
resourceService.delete(deleteResourceRequest);
return Result.success();
}
@Operation(summary = "queryResourceFileList", description = "QUERY_RESOURCE_FILE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class))})
@GetMapping(value = "/query-by-type")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<List<ResourceComponent>> queryResourceFileList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type) {
return Result.success(resourceService.queryResourceFiles(loginUser, type));
}
@Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100"))})
@GetMapping(value = "/view")
@ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR)
public Result<FetchFileContentResponse> viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsoluteFilePath,
@RequestParam(value = "skipLineNum") int skipLineNum,
@RequestParam(value = "limit") int limit) {
FetchFileContentRequest fetchFileContentRequest = FetchFileContentRequest.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath(resourceAbsoluteFilePath)
.limit(limit == -1 ? Integer.MAX_VALUE : skipLineNum)
.skipLineNum(skipLineNum)
.build();
return Result.success(resourceService.fetchResourceFileContent(fetchFileContentRequest));
}
/**
* download resource file
*
* @param loginUser login user
* @return resource content
*/
@Operation(summary = "downloadResource", description = "DOWNLOAD_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))})
@GetMapping(value = "/download")
@ResponseBody
@ApiException(DOWNLOAD_RESOURCE_FILE_ERROR)
public ResponseEntity downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName) throws Exception {
Resource file = resourceService.downloadResource(loginUser, fullName);
if (file == null) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(RESOURCE_NOT_EXIST.getMsg());
}
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + file.getFilename() + "\"")
.body(file);
public void downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
HttpServletResponse response,
@RequestParam(value = "fullName") String fileAbsolutePath) {
DownloadFileRequest downloadFileRequest = DownloadFileRequest.builder()
.loginUser(loginUser)
.fileAbsolutePath(fileAbsolutePath)
.build();
resourceService.downloadResource(response, downloadFileRequest);
}
/**
* create udf function
*
* @param loginUser login user
* @param type udf type
* @param funcName function name
* @param argTypes argument types
* @param database database
* @param description description
* @param className class name
* @return create result code
*/
@Operation(summary = "createUdfFunc", description = "CREATE_UDF_FUNCTION_NOTES")
@Parameters({
@Parameter(name = "type", description = "UDF_TYPE", required = true, schema = @Schema(implementation = UdfType.class)),
@ -612,8 +527,8 @@ public class ResourcesController extends BaseController {
@GetMapping(value = "/base-dir")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
public Result<Object> queryResourceBaseDir(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
public Result<String> queryResourceBaseDir(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type) {
return resourceService.queryResourceBaseDir(loginUser, type);
return Result.success(resourceService.queryResourceBaseDir(loginUser, type));
}
}

34
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceCreateRequest.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.Data;
import lombok.experimental.SuperBuilder;
@Data
@SuperBuilder
public abstract class AbstractResourceCreateRequest {
private User loginUser;
private String parentAbsoluteDirectory;
private ResourceType type;
}

29
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceDto.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import lombok.Data;
import lombok.experimental.SuperBuilder;
@Data
@SuperBuilder
public abstract class AbstractResourceDto {
private String resourceAbsolutePath;
}

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryDto.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CreateDirectoryDto {
private User loginUser;
private String directoryAbsolutePath;
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryRequest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateDirectoryRequest {
private User loginUser;
private ResourceType type;
private String parentAbsoluteDirectory;
private String directoryName;
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileDto.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileDto {
private User loginUser;
private String fileAbsolutePath;
private MultipartFile file;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentDto.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileFromContentDto {
private User loginUser;
private String fileAbsolutePath;
private String fileContent;
}

40
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentRequest.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileFromContentRequest {
private User loginUser;
private ResourceType type;
private String parentAbsoluteDirectory;
private String fileName;
private String fileContent;
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileRequest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileRequest {
private User loginUser;
private ResourceType type;
private String parentAbsoluteDirectory;
private String fileName;
private MultipartFile file;
}

36
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceDto.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DeleteResourceDto {
private User loginUser;
private String resourceAbsolutePath;
}

35
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceRequest.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DeleteResourceRequest {
private User loginUser;
private String resourceAbsolutePath;
}

36
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileDto.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DownloadFileDto {
private User loginUser;
private String fileAbsolutePath;
}

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileRequest.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DownloadFileRequest {
private User loginUser;
private String fileAbsolutePath;
}

38
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentDto.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class FetchFileContentDto {
private User loginUser;
private String resourceFileAbsolutePath;
private int skipLineNum;
private int limit;
}

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentRequest.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FetchFileContentRequest {
private User loginUser;
private String resourceFileAbsolutePath;
private int skipLineNum;
private int limit;
}

58
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/PagingResourceItemRequest.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PagingResourceItemRequest {
private User loginUser;
private String resourceAbsolutePath;
private ResourceType resourceType;
private String resourceNameKeyWord;
Integer pageNo;
Integer pageSize;
public void checkPageNoAndPageSize() {
if (pageNo <= 0) {
throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.PAGE_NUMBER);
}
if (pageSize <= 0) {
throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.PAGE_SIZE);
}
}
}

34
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/QueryResourceDto.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class QueryResourceDto {
private List<String> resourceAbsolutePaths;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryDto.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameDirectoryDto {
private User loginUser;
private String originDirectoryAbsolutePath;
private String targetDirectoryAbsolutePath;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryRequest.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameDirectoryRequest {
private User loginUser;
private String directoryAbsolutePath;
private String newDirectoryName;
}

38
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileDto.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameFileDto {
private User loginUser;
private String originFileAbsolutePath;
private String targetFileAbsolutePath;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileRequest.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameFileRequest {
private User loginUser;
private String fileAbsolutePath;
private String newFileName;
}

31
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
View File

@ -25,34 +25,13 @@ import java.util.List;
import lombok.Data;
import lombok.NoArgsConstructor;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
/**
* resource component
*/
@Data
@NoArgsConstructor
@JsonPropertyOrder({"id", "pid", "name", "fullName", "description", "isDirctory", "children", "type"})
public abstract class ResourceComponent {
public ResourceComponent(int id, String pid, String name, String fullName, String description, boolean isDirctory) {
this.id = id;
this.pid = pid;
this.name = name;
this.fullName = fullName;
this.isDirctory = isDirctory;
int directoryFlag = isDirctory ? 1 : 0;
this.idValue = String.format("%s_%s", id, directoryFlag);
}
/**
* id
*/
protected int id;
/**
* parent id
*/
protected String pid;
/**
* name
*/
@ -73,10 +52,7 @@ public abstract class ResourceComponent {
* is directory
*/
protected boolean isDirctory;
/**
* id value
*/
protected String idValue;
/**
* resoruce type
*/
@ -88,14 +64,11 @@ public abstract class ResourceComponent {
/**
* add resource component
*
* @param resourceComponent resource component
*/
public void add(ResourceComponent resourceComponent) {
children.add(resourceComponent);
}
public void setIdValue(int id, boolean isDirctory) {
int directoryFlag = isDirctory ? 1 : 0;
this.idValue = String.format("%s_%s", id, directoryFlag);
}
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileDto.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileDto {
private User loginUser;
private String fileAbsolutePath;
private MultipartFile file;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentDto.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileFromContentDto {
private User loginUser;
private String fileAbsolutePath;
private String fileContent;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentRequest.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileFromContentRequest {
private User loginUser;
private String fileAbsolutePath;
private String fileContent;
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileRequest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileRequest {
private User loginUser;
private String fileAbsolutePath;
private MultipartFile file;
}

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
View File

@ -122,12 +122,8 @@ public class ResourceTreeVisitor implements Visitor {
tempResourceComponent = new FileLeaf();
}
tempResourceComponent.setName(resource.getAlias());
// tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/",""));
tempResourceComponent.setName(resource.getFileName());
tempResourceComponent.setFullName(resource.getFullName());
tempResourceComponent.setId(resource.getId());
tempResourceComponent.setPid(resource.getPfullName());
tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory());
tempResourceComponent.setType(resource.getType());
return tempResourceComponent;
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ServiceException.java
View File

@ -52,4 +52,8 @@ public class ServiceException extends RuntimeException {
this.code = code;
}
public ServiceException(String message, Exception exception) {
this(Status.INTERNAL_SERVER_ERROR_ARGS, message, exception);
}
}

33
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java
View File

@ -31,14 +31,12 @@ import org.apache.dolphinscheduler.api.service.SchedulerService;
import org.apache.dolphinscheduler.api.service.TaskDefinitionService;
import org.apache.dolphinscheduler.api.service.TenantService;
import org.apache.dolphinscheduler.api.service.UsersService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ComplementDependentMode;
import org.apache.dolphinscheduler.common.enums.ExecutionOrder;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.enums.RunMode;
import org.apache.dolphinscheduler.common.enums.TaskDependType;
@ -614,25 +612,22 @@ public class PythonGateway {
* Get resource by given program type and full name. It returns map contain resource id, name.
* Useful in Python API create flink or spark task which need workflow information.
*
* @param programType program type one of SCALA, JAVA and PYTHON
* @param fullName full name of the resource
*/
public Map<String, Object> getResourcesFileInfo(String programType, String fullName) {
public Map<String, Object> getResourcesFileInfo(String fullName) {
Map<String, Object> result = new HashMap<>();
Result<Object> resources = resourceService.queryResourceByProgramType(dummyAdminUser, ResourceType.FILE,
ProgramType.valueOf(programType));
List<ResourceComponent> resourcesComponent = (List<ResourceComponent>) resources.getData();
List<ResourceComponent> namedResources =
resourcesComponent.stream().filter(s -> fullName.equals(s.getFullName())).collect(Collectors.toList());
List<ResourceComponent> resourceComponents =
resourceService.queryResourceFiles(dummyAdminUser, ResourceType.FILE);
List<ResourceComponent> namedResources = resourceComponents.stream()
.filter(s -> fullName.equals(s.getFullName()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(namedResources)) {
String msg =
String.format("Can not find valid resource by program type %s and name %s", programType, fullName);
String msg = String.format("Can not find valid resource by name %s", fullName);
log.error(msg);
throw new IllegalArgumentException(msg);
}
result.put("id", namedResources.get(0).getId());
result.put("name", namedResources.get(0).getName());
return result;
}
@ -671,20 +666,6 @@ public class PythonGateway {
return PythonGateway.class.getPackage().getImplementationVersion();
}
/**
* create or update resource.
* If the folder is not already created, it will be
*
* @param userName user who create or update resource
* @param fullName The fullname of resource.Includes path and suffix.
* @param resourceContent content of resource
* @return StorageEntity object which contains necessary information about resource
*/
public StorageEntity createOrUpdateResource(String userName, String fullName,
String resourceContent) throws Exception {
return resourceService.createOrUpdateResource(userName, fullName, resourceContent);
}
@PostConstruct
public void init() {
if (apiConfig.getPythonGateway().isEnabled()) {

201
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
View File

@ -17,181 +17,99 @@
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest;
import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.api.vo.ResourceItemVO;
import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.io.IOException;
import java.util.Map;
import java.util.List;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
/**
* resources service
*/
public interface ResourcesService {
/**
* create directory
*
* @param loginUser login user
* @param name alias
* @param type type
* @param pid parent id
* @param currentDir current directory
* @return create directory result
* Create a new directory in the resource storage, if the directory already exists will throw exception
*/
Result<Object> createDirectory(User loginUser,
String name,
ResourceType type,
int pid,
String currentDir);
void createDirectory(CreateDirectoryRequest createDirectoryRequest);
/**
* create resource
*
* @param loginUser login user
* @param name alias
* @param type type
* @param file file
* @param currentDir current directory
* @return create result code
* Rename the directory in the resource storage, if the origin directory not exists or the new directory already exists will throw exception.
* <p> If the origin directory is empty will only update the directory name.
* <p> If the origin directory is not empty will move all the files and directories to the new directory.
* <p> After update the origin directory will be deleted.
*/
Result<Object> uploadResource(User loginUser,
String name,
ResourceType type,
MultipartFile file,
String currentDir);
void renameDirectory(RenameDirectoryRequest renameDirectoryRequest);
/**
* update resource
* @param loginUser login user
* @param name name
* @param type resource type
* @param file resource file
* @return update result code
* Upload a new file to the resource storage, if the file already exists will throw exception
*/
Result<Object> updateResource(User loginUser,
String fullName,
String tenantCode,
String name,
ResourceType type,
MultipartFile file);
void createFile(CreateFileRequest createFileRequest);
/**
* query resources list paging
*
* @param loginUser login user
* @param type resource type
* @param searchVal search value
* @param pageNo page number
* @param pageSize page size
* @return resource list page
* Update the file in the resource storage, if the origin file not exists or the new file already exists will throw exception.
* <p> If the new file is empty will only update the file name.
* <p> If the new file is not empty will update the file content and name.
* <p> After update the origin file will be deleted.
*/
Result<PageInfo<StorageEntity>> queryResourceListPaging(User loginUser, String fullName, String resTenantCode,
ResourceType type, String searchVal, Integer pageNo,
Integer pageSize);
void updateFile(UpdateFileRequest updateFileRequest);
/**
* query resource list
*
* @param loginUser login user
* @param type resource type
* @return resource list
* Rename the file in the resource storage, if the origin file not exists or the new file already exists will throw exception.
*/
Map<String, Object> queryResourceList(User loginUser, ResourceType type, String fullName);
void renameFile(RenameFileRequest renameFileRequest);
/**
* query resource list by program type
*
* @param loginUser login user
* @param type resource type
* @return resource list
* Create a new file in the resource storage, if the file already exists will throw exception.
* Different with {@link ResourcesService#createFile(CreateFileRequest)} this method will create a new file with the given content.
*/
Result<Object> queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType);
void createFileFromContent(CreateFileFromContentRequest createFileFromContentRequest);
/**
* delete resource
*
* @param loginUser login user
* @return delete result code
* @throws IOException exception
* Update the file content.
*/
Result<Object> delete(User loginUser, String fullName, String tenantCode) throws IOException;
void updateFileFromContent(UpdateFileFromContentRequest updateFileContentRequest);
/**
* verify resource by name and type
* @param loginUser login user
* @param fullName resource full name
* @param type resource type
* @return true if the resource name not exists, otherwise return false
* Paging query resource items.
* <p>If the login user is not admin will only query the resource items that under the user's tenant.
* <p>If the login user is admin and {@link PagingResourceItemRequest##resourceAbsolutePath} is null will return all the resource items.
*/
Result<Object> verifyResourceName(String fullName, ResourceType type, User loginUser);
PageInfo<ResourceItemVO> pagingResourceItem(PagingResourceItemRequest pagingResourceItemRequest);
/**
* verify resource by file name
* @param fileName resource file name
* @param type resource type
* @return true if the resource file name, otherwise return false
* Query the resource file items by the given resource type and program type.
*/
Result<Object> queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode);
List<ResourceComponent> queryResourceFiles(User loginUser, ResourceType type);
/**
* view resource file online
*
* @param skipLineNum skip line number
* @param limit limit
* @param fullName fullName
* @return resource content
* Delete the resource item.
* <p>If the resource item is a directory will delete all the files and directories under the directory.
* <p>If the resource item is a file will delete the file.
* <p>If the resource item not exists will throw exception.
*/
Result<Object> readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit);
void delete(DeleteResourceRequest deleteResourceRequest);
/**
* create resource file online
*
* @param loginUser login user
* @param type resource type
* @param fileName file name
* @param fileSuffix file suffix
* @param content content
* @return create result code
* Fetch the file content.
*/
Result<Object> createResourceFile(User loginUser, ResourceType type, String fileName, String fileSuffix,
String content, String currentDirectory);
FetchFileContentResponse fetchResourceFileContent(FetchFileContentRequest fetchFileContentRequest);
/**
* create or update resource.
* If the folder is not already created, it will be ignored and directly create the new file
*
* @param userName user who create or update resource
* @param fullName The fullname of resource.Includes path and suffix.
* @param resourceContent content of resource
*/
StorageEntity createOrUpdateResource(String userName, String fullName, String resourceContent) throws Exception;
/**
* updateProcessInstance resource
*
* @param loginUser login user
* @param fullName full name
* @param tenantCode tenantCode
* @param content content
* @return update result cod
*/
Result<Object> updateResourceContent(User loginUser, String fullName, String tenantCode,
String content);
/**
* download file
*
* @return resource content
* @throws IOException exception
*/
org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException;
void downloadResource(HttpServletResponse response, DownloadFileRequest downloadFileRequest);
/**
* Get resource by given resource type and file name.
@ -202,21 +120,6 @@ public interface ResourcesService {
*/
StorageEntity queryFileStatus(String userName, String fileName) throws Exception;
/**
* delete DATA_TRANSFER data in resource center
*
* @param loginUser user who query resource
* @param days number of days
*/
DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer days);
/**
* get resource base dir
*
* @param loginUser login user
* @param type resource type
* @return
*/
Result<Object> queryResourceBaseDir(User loginUser, ResourceType type);
String queryResourceBaseDir(User loginUser, ResourceType type);
}

1562
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
View File

File diff suppressed because it is too large Load Diff

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
View File

@ -39,7 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@ -84,7 +84,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
private QueueService queueService;
@Autowired(required = false)
private StorageOperate storageOperate;
private StorageOperator storageOperator;
/**
* Check the tenant new object valid or not
@ -136,14 +136,13 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
* @param queueId queue id
* @param desc description
* @return create result code
* @throws Exception exception
*/
@Override
@Transactional(rollbackFor = Exception.class)
public Tenant createTenant(User loginUser,
String tenantCode,
int queueId,
String desc) throws Exception {
String desc) {
if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_CREATE)) {
throw new ServiceException(Status.USER_NO_OPERATION_PERM);
}
@ -154,7 +153,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
createTenantValid(tenant);
tenantMapper.insert(tenant);
storageOperate.createTenantDirIfNotExists(tenantCode);
return tenant;
}
@ -209,11 +207,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
updateTenantValid(existsTenant, updateTenant);
updateTenant.setCreateTime(existsTenant.getCreateTime());
// updateProcessInstance tenant
// if the tenant code is modified, the original resource needs to be copied to the new tenant.
if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode())) {
storageOperate.createTenantDirIfNotExists(tenantCode);
}
int update = tenantMapper.updateById(updateTenant);
if (update <= 0) {
throw new ServiceException(Status.UPDATE_TENANT_ERROR);
@ -262,7 +255,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
}
processInstanceMapper.updateProcessInstanceByTenantCode(tenant.getTenantCode(), Constants.DEFAULT);
storageOperate.deleteTenant(tenant.getTenantCode());
}
private List<ProcessInstance> getProcessInstancesByTenant(Tenant tenant) {

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
View File

@ -28,11 +28,10 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -62,7 +61,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
private UDFUserMapper udfUserMapper;
@Autowired(required = false)
private StorageOperate storageOperate;
private StorageOperator storageOperator;
/**
* create udf function
@ -107,12 +106,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
return result;
}
Boolean existResource = false;
try {
existResource = storageOperate.exists(fullName);
} catch (IOException e) {
log.error("Check resource error: {}", fullName, e);
}
boolean existResource = storageOperator.exists(fullName);
if (!existResource) {
log.error("resource full name {} is not exist", fullName);
@ -241,7 +235,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
Boolean doesResExist = false;
try {
doesResExist = storageOperate.exists(fullName);
doesResExist = storageOperator.exists(fullName);
} catch (Exception e) {
log.error("udf resource :{} checking error", fullName, e);
result.setCode(Status.RESOURCE_NOT_EXIST.getCode());

56
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
View File

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service.impl;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.USER_MANAGER;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.MetricsCleanUpService;
@ -50,7 +49,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@ -110,7 +109,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
private ProjectMapper projectMapper;
@Autowired(required = false)
private StorageOperate storageOperate;
private StorageOperator storageOperator;
@Autowired
private K8sNamespaceUserMapper k8sNamespaceUserMapper;
@ -171,9 +170,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
User user = createUser(userName, userPassword, email, tenantId, phone, queue, state);
Tenant tenant = tenantMapper.queryById(tenantId);
storageOperate.createTenantDirIfNotExists(tenant.getTenantCode());
log.info("User is created and id is {}.", user.getId());
result.put(Constants.DATA_LIST, user);
putMsg(result, Status.SUCCESS);
@ -1128,54 +1124,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
return msg;
}
/**
* copy resource files
* xxx unchecked
*
* @param resourceComponent resource component
* @param srcBasePath src base path
* @param dstBasePath dst base path
* @throws IOException io exception
*/
private void copyResourceFiles(String oldTenantCode, String newTenantCode, ResourceComponent resourceComponent,
String srcBasePath, String dstBasePath) {
List<ResourceComponent> components = resourceComponent.getChildren();
try {
if (CollectionUtils.isNotEmpty(components)) {
for (ResourceComponent component : components) {
// verify whether exist
if (!storageOperate.exists(
String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()))) {
log.error("Resource file: {} does not exist, copy error.", component.getFullName());
throw new ServiceException(Status.RESOURCE_NOT_EXIST);
}
if (!component.isDirctory()) {
// copy it to dst
storageOperate.copy(String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()),
String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()), false, true);
continue;
}
if (CollectionUtils.isEmpty(component.getChildren())) {
// if not exist,need create it
if (!storageOperate
.exists(String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) {
storageOperate.mkdir(newTenantCode,
String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()));
}
} else {
copyResourceFiles(oldTenantCode, newTenantCode, component, srcBasePath, dstBasePath);
}
}
}
} catch (IOException e) {
log.error("copy the resources failed,the error message is {}", e.getMessage());
}
}
/**
* registry user, default state is 0, default tenant_id is 1, no phone, no queue
*

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java
View File

@ -20,12 +20,16 @@ package org.apache.dolphinscheduler.api.utils;
import java.util.Collections;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Setter;
import com.baomidou.mybatisplus.core.metadata.IPage;
@Data
@Builder
@AllArgsConstructor
public class PageInfo<T> {
/**

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator;
public interface ITransformer<T, R> {
R transform(T t);
}

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator;
public interface IValidator<T> {
void validate(T t);
}

56
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import lombok.AllArgsConstructor;
@AllArgsConstructor
public abstract class AbstractResourceTransformer<T, R> implements ITransformer<T, R> {
protected TenantDao tenantDao;
protected StorageOperator storageOperator;
protected String getParentDirectoryAbsolutePath(User loginUser, String parentAbsoluteDirectory, ResourceType type) {
String tenantCode = tenantDao.queryOptionalById(loginUser.getTenantId())
.orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST))
.getTenantCode();
String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, type);
// If the parent directory is / then will transform to userResRootPath
// This only happens when the front-end go into the resource page first
// todo: we need to change the front-end logic to avoid this
if (parentAbsoluteDirectory.equals("/")) {
return userResRootPath;
}
if (!StringUtils.startsWith(parentAbsoluteDirectory, userResRootPath)) {
throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentAbsoluteDirectory);
}
return parentAbsoluteDirectory;
}
}

138
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.AbstractResourceDto;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.validator.IValidator;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.springframework.web.multipart.MultipartFile;
import com.google.common.io.Files;
public abstract class AbstractResourceValidator<T> implements IValidator<T> {
private static final Set<String> FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT = new HashSet<>(Arrays.asList(
StringUtils.defaultIfBlank(FileUtils.getResourceViewSuffixes(), "").split(",")));
protected final StorageOperator storageOperator;
private final TenantDao tenantDao;
public AbstractResourceValidator(StorageOperator storageOperator, TenantDao tenantDao) {
this.storageOperator = storageOperator;
this.tenantDao = tenantDao;
}
public void exceptionResourceAbsolutePathInvalidated(String resourceAbsolutePath) {
if (StringUtils.isBlank(resourceAbsolutePath)) {
throw new ServiceException("The resource path is null");
}
if (!resourceAbsolutePath.startsWith(storageOperator.getStorageBaseDirectory())) {
throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
}
if (resourceAbsolutePath.contains("..")) {
throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
}
}
public void exceptionFileInvalidated(MultipartFile file) {
if (file == null) {
throw new ServiceException("The file is null");
}
}
public void exceptionFileContentInvalidated(String fileContent) {
if (StringUtils.isEmpty(fileContent)) {
throw new ServiceException("The file content is null");
}
}
public void exceptionFileContentCannotFetch(String fileAbsolutePath) {
String fileExtension = Files.getFileExtension(fileAbsolutePath);
if (!FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT.contains(fileExtension)) {
throw new ServiceException("The file type: " + fileExtension + " cannot be fetched");
}
}
public void exceptionResourceNotExists(String resourceAbsolutePath) {
if (!storageOperator.exists(resourceAbsolutePath)) {
throw new ServiceException("Thr resource is not exists: " + resourceAbsolutePath);
}
}
public void exceptionResourceExists(String resourceAbsolutePath) {
if (storageOperator.exists(resourceAbsolutePath)) {
throw new ServiceException("The resource is already exist: " + resourceAbsolutePath);
}
}
public void exceptionResourceIsNotDirectory(String resourceAbsolutePath) {
if (StringUtils.isNotEmpty(Files.getFileExtension(resourceAbsolutePath))) {
throw new ServiceException("The path is not a directory: " + resourceAbsolutePath);
}
}
public void exceptionResourceIsNotFile(String fileAbsolutePath) {
if (StringUtils.isEmpty(Files.getFileExtension(fileAbsolutePath))) {
throw new ServiceException("The path is not a file: " + fileAbsolutePath);
}
}
public void exceptionUserNoResourcePermission(User user, AbstractResourceDto resourceDto) {
exceptionUserNoResourcePermission(user, resourceDto.getResourceAbsolutePath());
}
public void exceptionUserNoResourcePermission(User user, String resourceAbsolutePath) {
if (user.getUserType() == UserType.ADMIN_USER) {
return;
}
// check if the user have resource tenant permission
// Parse the resource path to get the tenant code
ResourceMetadata resourceMetaData = storageOperator.getResourceMetaData(resourceAbsolutePath);
if (!resourceAbsolutePath.startsWith(resourceMetaData.getResourceBaseDirectory())) {
throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
}
// todo: inject the tenant when login
Tenant tenant = tenantDao.queryOptionalById(user.getTenantId())
.orElseThrow(() -> new ServiceException(Status.TENANT_NOT_EXIST, user.getTenantId()));
String userTenant = tenant.getTenantCode();
if (!userTenant.equals(resourceMetaData.getTenant())) {
throw new ServiceException(
"The user's tenant is " + userTenant + " have no permission to access the resource: "
+ resourceAbsolutePath);
}
}
}

50
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import com.google.common.io.Files;
@Component
public class CreateDirectoryDtoValidator extends AbstractResourceValidator<CreateDirectoryDto> {
public CreateDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(CreateDirectoryDto createDirectoryDto) {
String directoryAbsolutePath = createDirectoryDto.getDirectoryAbsolutePath();
exceptionResourceAbsolutePathInvalidated(directoryAbsolutePath);
exceptionResourceExists(directoryAbsolutePath);
exceptionUserNoResourcePermission(createDirectoryDto.getLoginUser(), directoryAbsolutePath);
exceptionResourceIsNotDirectory(directoryAbsolutePath);
if (StringUtils.isNotEmpty(Files.getFileExtension(directoryAbsolutePath))) {
throw new ServiceException("The path is not a directory: " + directoryAbsolutePath);
}
}
}

87
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class CreateDirectoryRequestTransformer implements ITransformer<CreateDirectoryRequest, CreateDirectoryDto> {
@Autowired
private TenantDao tenantDao;
@Autowired
private StorageOperator storageOperator;
@Override
public CreateDirectoryDto transform(CreateDirectoryRequest createDirectoryRequest) {
validateCreateDirectoryRequest(createDirectoryRequest);
return doTransform(createDirectoryRequest);
}
private CreateDirectoryDto doTransform(CreateDirectoryRequest createDirectoryRequest) {
String directoryAbsolutePath = getDirectoryAbsolutePath(createDirectoryRequest);
return CreateDirectoryDto.builder()
.loginUser(createDirectoryRequest.getLoginUser())
.directoryAbsolutePath(directoryAbsolutePath)
.build();
}
private void validateCreateDirectoryRequest(CreateDirectoryRequest createDirectoryRequest) {
checkNotNull(createDirectoryRequest.getLoginUser(), "loginUser is null");
checkNotNull(createDirectoryRequest.getType(), "resource type is null");
checkNotNull(createDirectoryRequest.getDirectoryName(), "directory name is null");
checkNotNull(createDirectoryRequest.getParentAbsoluteDirectory(), "parent directory is null");
}
private String getDirectoryAbsolutePath(CreateDirectoryRequest createDirectoryRequest) {
String tenantCode = tenantDao.queryOptionalById(createDirectoryRequest.getLoginUser().getTenantId())
.orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST))
.getTenantCode();
String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, createDirectoryRequest.getType());
String parentDirectoryName = createDirectoryRequest.getParentAbsoluteDirectory();
String directoryName = createDirectoryRequest.getDirectoryName();
// If the parent directory is / then will transform to userResRootPath
// This only happens when the front-end go into the resource page first
// todo: we need to change the front-end logic to avoid this
if (parentDirectoryName.equals("/")) {
return FileUtils.concatFilePath(userResRootPath, directoryName);
}
if (!StringUtils.startsWith(parentDirectoryName, userResRootPath)) {
throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentDirectoryName);
}
return FileUtils.concatFilePath(parentDirectoryName, directoryName);
}
}

47
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
@Component
public class CreateFileDtoValidator extends AbstractResourceValidator<CreateFileDto> {
public CreateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(CreateFileDto createFileDto) {
String fileAbsolutePath = createFileDto.getFileAbsolutePath();
User loginUser = createFileDto.getLoginUser();
MultipartFile file = createFileDto.getFile();
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceExists(fileAbsolutePath);
exceptionFileInvalidated(file);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
}
}

46
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class CreateFileFromContentDtoValidator extends AbstractResourceValidator<CreateFileFromContentDto> {
public CreateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(CreateFileFromContentDto createFileFromContentDto) {
String fileAbsolutePath = createFileFromContentDto.getFileAbsolutePath();
User loginUser = createFileFromContentDto.getLoginUser();
String fileContent = createFileFromContentDto.getFileContent();
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionResourceExists(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionFileContentInvalidated(fileContent);
}
}

43
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class DeleteResourceDtoValidator extends AbstractResourceValidator<DeleteResourceDto> {
public DeleteResourceDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(DeleteResourceDto deleteResourceDto) {
String resourceAbsolutePath = deleteResourceDto.getResourceAbsolutePath();
User loginUser = deleteResourceDto.getLoginUser();
exceptionResourceAbsolutePathInvalidated(resourceAbsolutePath);
exceptionResourceNotExists(resourceAbsolutePath);
exceptionUserNoResourcePermission(loginUser, resourceAbsolutePath);
}
}

44
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class DownloadFileDtoValidator extends AbstractResourceValidator<DownloadFileDto> {
public DownloadFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(DownloadFileDto downloadFileDto) {
String fileAbsolutePath = downloadFileDto.getFileAbsolutePath();
User loginUser = downloadFileDto.getLoginUser();
exceptionResourceNotExists(fileAbsolutePath);
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
}
}

48
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class FetchFileContentDtoValidator extends AbstractResourceValidator<FetchFileContentDto> {
public FetchFileContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(FetchFileContentDto fetchFileContentDto) {
if (fetchFileContentDto.getSkipLineNum() < 0) {
throw new ServiceException("skipLineNum must be greater than or equal to 0");
}
String resourceFileAbsolutePath = fetchFileContentDto.getResourceFileAbsolutePath();
User loginUser = fetchFileContentDto.getLoginUser();
exceptionResourceAbsolutePathInvalidated(resourceFileAbsolutePath);
exceptionResourceIsNotFile(resourceFileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, resourceFileAbsolutePath);
exceptionFileContentCannotFetch(resourceFileAbsolutePath);
}
}

71
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class FileFromContentRequestTransformer
extends
AbstractResourceTransformer<CreateFileFromContentRequest, CreateFileFromContentDto> {
public FileFromContentRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) {
super(tenantDao, storageOperator);
}
@Override
public CreateFileFromContentDto transform(CreateFileFromContentRequest createFileFromContentRequest) {
validateCreateFileRequest(createFileFromContentRequest);
return doTransform(createFileFromContentRequest);
}
private void validateCreateFileRequest(CreateFileFromContentRequest createFileFromContentRequest) {
checkNotNull(createFileFromContentRequest.getLoginUser(), "loginUser is null");
checkNotNull(createFileFromContentRequest.getType(), "resource type is null");
checkNotNull(createFileFromContentRequest.getFileName(), "file name is null");
checkNotNull(createFileFromContentRequest.getParentAbsoluteDirectory(), "parent directory is null");
checkNotNull(createFileFromContentRequest.getFileContent(), "file content is null");
}
private CreateFileFromContentDto doTransform(CreateFileFromContentRequest createFileFromContentRequest) {
String fileAbsolutePath = getFileAbsolutePath(createFileFromContentRequest);
return CreateFileFromContentDto.builder()
.loginUser(createFileFromContentRequest.getLoginUser())
.fileAbsolutePath(fileAbsolutePath)
.fileContent(createFileFromContentRequest.getFileContent())
.build();
}
private String getFileAbsolutePath(CreateFileFromContentRequest createFileFromContentRequest) {
String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath(
createFileFromContentRequest.getLoginUser(),
createFileFromContentRequest.getParentAbsoluteDirectory(),
createFileFromContentRequest.getType());
return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileFromContentRequest.getFileName());
}
}

68
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class FileRequestTransformer extends AbstractResourceTransformer<CreateFileRequest, CreateFileDto> {
public FileRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) {
super(tenantDao, storageOperator);
}
@Override
public CreateFileDto transform(CreateFileRequest createFileRequest) {
validateCreateFileRequest(createFileRequest);
return doTransform(createFileRequest);
}
private void validateCreateFileRequest(CreateFileRequest createFileRequest) {
checkNotNull(createFileRequest.getLoginUser(), "loginUser is null");
checkNotNull(createFileRequest.getType(), "resource type is null");
checkNotNull(createFileRequest.getFileName(), "file name is null");
checkNotNull(createFileRequest.getParentAbsoluteDirectory(), "parent directory is null");
checkNotNull(createFileRequest.getFile(), "file is null");
}
private CreateFileDto doTransform(CreateFileRequest createFileRequest) {
String fileAbsolutePath = getFileAbsolutePath(createFileRequest);
return CreateFileDto.builder()
.loginUser(createFileRequest.getLoginUser())
.file(createFileRequest.getFile())
.fileAbsolutePath(fileAbsolutePath)
.build();
}
private String getFileAbsolutePath(CreateFileRequest createFileRequest) {
String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath(
createFileRequest.getLoginUser(),
createFileRequest.getParentAbsoluteDirectory(),
createFileRequest.getType());
return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileRequest.getFileName());
}
}

90
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.stream.Collectors;
import lombok.AllArgsConstructor;
import org.springframework.stereotype.Component;
import com.google.common.collect.Lists;
@Component
@AllArgsConstructor
public class PagingResourceItemRequestTransformer implements ITransformer<PagingResourceItemRequest, QueryResourceDto> {
private final StorageOperator storageOperator;
private final TenantDao tenantDao;
@Override
public QueryResourceDto transform(PagingResourceItemRequest pagingResourceItemRequest) {
validatePagingResourceItemRequest(pagingResourceItemRequest);
if (StringUtils.isNotEmpty(pagingResourceItemRequest.getResourceAbsolutePath())) {
// query from the given path
return QueryResourceDto.builder()
.resourceAbsolutePaths(Lists.newArrayList(pagingResourceItemRequest.getResourceAbsolutePath()))
.build();
}
ResourceType resourceType = pagingResourceItemRequest.getResourceType();
User loginUser = pagingResourceItemRequest.getLoginUser();
if (loginUser.getUserType() == UserType.ADMIN_USER) {
// If the current user is admin
// then will query all tenant resources
List<String> resourceAbsolutePaths = tenantDao.queryAll()
.stream()
.map(tenant -> storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType))
.collect(Collectors.toList());
return QueryResourceDto.builder()
.resourceAbsolutePaths(resourceAbsolutePaths)
.build();
} else {
// todo: inject the tenantCode when login
Tenant tenant = tenantDao.queryById(loginUser.getTenantId());
String storageBaseDirectory = storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType);
return QueryResourceDto.builder()
.resourceAbsolutePaths(Lists.newArrayList(storageBaseDirectory))
.build();
}
}
private void validatePagingResourceItemRequest(PagingResourceItemRequest pagingResourceItemRequest) {
checkNotNull(pagingResourceItemRequest.getLoginUser(), "loginUser is null");
checkNotNull(pagingResourceItemRequest.getResourceType(), "resourceType is null");
}
}

50
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class RenameDirectoryDtoValidator extends AbstractResourceValidator<RenameDirectoryDto> {
public RenameDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(RenameDirectoryDto renameDirectoryDto) {
String originDirectoryAbsolutePath = renameDirectoryDto.getOriginDirectoryAbsolutePath();
User loginUser = renameDirectoryDto.getLoginUser();
String targetDirectoryAbsolutePath = renameDirectoryDto.getTargetDirectoryAbsolutePath();
exceptionResourceAbsolutePathInvalidated(originDirectoryAbsolutePath);
exceptionResourceIsNotDirectory(originDirectoryAbsolutePath);
exceptionResourceNotExists(originDirectoryAbsolutePath);
exceptionUserNoResourcePermission(loginUser, originDirectoryAbsolutePath);
exceptionResourceAbsolutePathInvalidated(targetDirectoryAbsolutePath);
exceptionResourceIsNotDirectory(targetDirectoryAbsolutePath);
exceptionResourceExists(targetDirectoryAbsolutePath);
exceptionUserNoResourcePermission(loginUser, targetDirectoryAbsolutePath);
}
}

56
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class RenameDirectoryRequestTransformer implements ITransformer<RenameDirectoryRequest, RenameDirectoryDto> {
@Override
public RenameDirectoryDto transform(RenameDirectoryRequest renameDirectoryRequest) {
String originDirectoryAbsolutePath = renameDirectoryRequest.getDirectoryAbsolutePath();
String targetDirectoryName = renameDirectoryRequest.getNewDirectoryName();
String targetDirectoryAbsolutePath =
getTargetDirectoryAbsolutePath(originDirectoryAbsolutePath, targetDirectoryName);
return RenameDirectoryDto.builder()
.loginUser(renameDirectoryRequest.getLoginUser())
.originDirectoryAbsolutePath(originDirectoryAbsolutePath)
.targetDirectoryAbsolutePath(targetDirectoryAbsolutePath)
.build();
}
private String getTargetDirectoryAbsolutePath(String originDirectoryAbsolutePath, String targetDirectoryName) {
String originDirectoryParentAbsolutePath = StringUtils.substringBeforeLast(
originDirectoryAbsolutePath, File.separator);
return originDirectoryParentAbsolutePath + File.separator + targetDirectoryName;
}
}

50
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class RenameFileDtoValidator extends AbstractResourceValidator<RenameFileDto> {
public RenameFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(RenameFileDto renameFileDto) {
String originFileAbsolutePath = renameFileDto.getOriginFileAbsolutePath();
User loginUser = renameFileDto.getLoginUser();
String targetFileAbsolutePath = renameFileDto.getTargetFileAbsolutePath();
exceptionResourceAbsolutePathInvalidated(originFileAbsolutePath);
exceptionResourceNotExists(originFileAbsolutePath);
exceptionResourceIsNotFile(originFileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, originFileAbsolutePath);
exceptionResourceAbsolutePathInvalidated(targetFileAbsolutePath);
exceptionResourceExists(targetFileAbsolutePath);
exceptionResourceIsNotFile(targetFileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, targetFileAbsolutePath);
}
}

47
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class RenameFileRequestTransformer implements ITransformer<RenameFileRequest, RenameFileDto> {
@Autowired
private StorageOperator storageOperator;
@Override
public RenameFileDto transform(RenameFileRequest renameFileRequest) {
ResourceMetadata resourceMetaData =
storageOperator.getResourceMetaData(renameFileRequest.getFileAbsolutePath());
return RenameFileDto.builder()
.loginUser(renameFileRequest.getLoginUser())
.originFileAbsolutePath(renameFileRequest.getFileAbsolutePath())
.targetFileAbsolutePath(FileUtils.concatFilePath(resourceMetaData.getResourceParentAbsolutePath(),
renameFileRequest.getNewFileName()))
.build();
}
}

57
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import java.util.Objects;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import com.google.common.io.Files;
@Component
public class UpdateFileDtoValidator extends AbstractResourceValidator<UpdateFileDto> {
public UpdateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(UpdateFileDto updateFileDto) {
String fileAbsolutePath = updateFileDto.getFileAbsolutePath();
User loginUser = updateFileDto.getLoginUser();
MultipartFile file = updateFileDto.getFile();
if (!Objects.equals(Files.getFileExtension(file.getName()),
Files.getFileExtension(updateFileDto.getFileAbsolutePath()))) {
throw new ServiceException("file extension cannot not change");
}
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceNotExists(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionFileInvalidated(file);
}
}

47
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class UpdateFileFromContentDtoValidator extends AbstractResourceValidator<UpdateFileFromContentDto> {
public UpdateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(UpdateFileFromContentDto updateFileFromContentDto) {
String fileAbsolutePath = updateFileFromContentDto.getFileAbsolutePath();
User loginUser = updateFileFromContentDto.getLoginUser();
String fileContent = updateFileFromContentDto.getFileContent();
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceNotExists(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionFileContentCannotFetch(fileAbsolutePath);
exceptionFileContentInvalidated(fileContent);
}
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.springframework.stereotype.Component;
@Component
public class UpdateFileFromContentRequestTransformer
implements
ITransformer<UpdateFileFromContentRequest, UpdateFileFromContentDto> {
@Override
public UpdateFileFromContentDto transform(UpdateFileFromContentRequest updateFileContentRequest) {
return UpdateFileFromContentDto.builder()
.loginUser(updateFileContentRequest.getLoginUser())
.fileAbsolutePath(updateFileContentRequest.getFileAbsolutePath())
.fileContent(updateFileContentRequest.getFileContent())
.build();
}
}

38
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.springframework.stereotype.Component;
@Component
public class UpdateFileRequestTransformer implements ITransformer<UpdateFileRequest, UpdateFileDto> {
@Override
public UpdateFileDto transform(UpdateFileRequest updateFileRequest) {
return UpdateFileDto.builder()
.loginUser(updateFileRequest.getLoginUser())
.fileAbsolutePath(updateFileRequest.getFileAbsolutePath())
.file(updateFileRequest.getFile())
.build();
}
}

74
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.vo;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.util.Date;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ResourceItemVO {
// todo: remove this field, directly use fileName
private String alias;
// todo: use tenantName instead of userName
private String userName;
private String fileName;
private String fullName;
private boolean isDirectory;
private ResourceType type;
private long size;
private Date createTime;
private Date updateTime;
public ResourceItemVO(StorageEntity storageEntity) {
this.isDirectory = storageEntity.isDirectory();
this.alias = storageEntity.getFileName();
this.fileName = storageEntity.getFileName();
this.fullName = storageEntity.getFullName();
this.type = storageEntity.getType();
this.size = storageEntity.getSize();
this.createTime = storageEntity.getCreateTime();
this.updateTime = storageEntity.getUpdateTime();
if (isDirectory) {
alias = StringUtils.removeEndIgnoreCase(alias, File.separator);
fileName = StringUtils.removeEndIgnoreCase(fileName, File.separator);
fullName = StringUtils.removeEndIgnoreCase(fullName, File.separator);
}
}
}

33
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.vo.resources;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FetchFileContentResponse {
private String content;
}

5
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java
View File

@ -27,6 +27,11 @@ import org.junit.jupiter.api.function.Executable;
public class AssertionsHelper extends Assertions {
public static void assertThrowServiceException(String message, Executable executable) {
ServiceException exception = Assertions.assertThrows(ServiceException.class, executable);
Assertions.assertEquals(message, exception.getMessage());
}
public static void assertThrowsServiceException(Status status, Executable executable) {
ServiceException exception = Assertions.assertThrows(ServiceException.class, executable);
Assertions.assertEquals(status.getCode(), exception.getCode());

143
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.api.controller;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
@ -28,29 +29,24 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
/**
* resources controller test
*/
import com.fasterxml.jackson.core.type.TypeReference;
public class ResourcesControllerTest extends AbstractControllerTest {
private static final Logger logger = LoggerFactory.getLogger(ResourcesControllerTest.class);
@ -61,37 +57,12 @@ public class ResourcesControllerTest extends AbstractControllerTest {
@MockBean(name = "udfFuncServiceImpl")
private UdfFuncService udfFuncService;
@Test
public void testQuerytResourceList() throws Exception {
Map<String, Object> mockResult = new HashMap<>();
mockResult.put(Constants.STATUS, Status.SUCCESS);
Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any(), Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("type", ResourceType.FILE.name());
MvcResult mvcResult = mockMvc.perform(get("/resources/list")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryResourceListPaging() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.SUCCESS.getCode());
Mockito.when(resourcesService.queryResourceListPaging(
Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
.thenReturn(mockResult);
// Mockito.when(resourcesService.pagingResourceItem()
// .thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("type", String.valueOf(ResourceType.FILE));
@ -111,41 +82,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testVerifyResourceName() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(resourcesService.verifyResourceName(Mockito.anyString(), Mockito.any(), Mockito.any()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "list_resources_1.sh");
paramsMap.add("type", "FILE");
MvcResult mvcResult = mockMvc.perform(get("/resources/verify-name")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testViewResource() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.HDFS_NOT_STARTUP.getCode());
Mockito.when(resourcesService.readResource(Mockito.any(),
Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
.thenReturn(mockResult);
FetchFileContentResponse fetchFileContentResponse = FetchFileContentResponse.builder()
.content("echo hello")
.build();
Mockito.when(resourcesService.fetchResourceFileContent(Mockito.any()))
.thenReturn(fetchFileContentResponse);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("skipLineNum", "2");
@ -160,19 +107,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Result<FetchFileContentResponse> result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(),
new TypeReference<Result<FetchFileContentResponse>>() {
});
Assertions.assertEquals(Status.HDFS_NOT_STARTUP.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
assertEquals(fetchFileContentResponse, result.getData());
}
@Test
public void testCreateResourceFile() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(resourcesService.createResourceFile(Mockito.any(), Mockito.any(), Mockito.anyString(),
Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
.thenReturn(mockResult);
Mockito.doNothing().when(resourcesService).createFileFromContent(Mockito.any());
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("type", String.valueOf(ResourceType.FILE));
@ -190,19 +135,16 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Result<Void> result =
JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference<Result<Void>>() {
});
Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
@Test
public void testUpdateResourceContent() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyString(),
Mockito.anyString(), Mockito.anyString()))
.thenReturn(mockResult);
Mockito.doNothing().when(resourcesService).updateFileFromContent(Mockito.any());
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("id", "1");
@ -217,17 +159,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Result<Void> result =
JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference<Result<Void>>() {
});
Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
@Test
public void testDownloadResource() throws Exception {
Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyString()))
.thenReturn(null);
Mockito.doNothing().when(resourcesService).downloadResource(Mockito.any(), Mockito.any());
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
@ -235,7 +177,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
MvcResult mvcResult = mockMvc.perform(get("/resources/download")
.params(paramsMap)
.header(SESSION_ID, sessionId))
.andExpect(status().is(HttpStatus.BAD_REQUEST.value()))
.andExpect(status().isOk())
.andReturn();
Assertions.assertNotNull(mvcResult);
@ -269,7 +211,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -289,7 +231,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -323,7 +265,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -348,7 +290,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -370,7 +312,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -392,7 +334,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -410,17 +352,13 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testDeleteResource() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.SUCCESS.getCode());
Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyString(),
Mockito.anyString()))
.thenReturn(mockResult);
Mockito.doNothing().when(resourcesService).delete(Mockito.any());
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("fullName", "dolphinscheduler/resourcePath");
paramsMap.add("tenantCode", "123");
@ -431,9 +369,10 @@ public class ResourcesControllerTest extends AbstractControllerTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Result<Void> result =
JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference<Result<Void>>() {
});
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
}

16
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java
View File

@ -97,19 +97,6 @@ public class PythonGatewayTest {
Assertions.assertEquals((long) result.get("taskDefinitionCode"), taskDefinition.getCode());
}
@Test
public void testCreateResource() {
User user = getTestUser();
String resourceDir = "/dir1/dir2/";
String resourceName = "test";
String resourceSuffix = "py";
String content = "content";
String resourceFullName = resourceDir + resourceName + "." + resourceSuffix;
Assertions.assertDoesNotThrow(
() -> pythonGateway.createOrUpdateResource(user.getUserName(), resourceFullName, content));
}
@Test
public void testQueryResourcesFileInfo() throws Exception {
User user = getTestUser();
@ -118,12 +105,11 @@ public class PythonGatewayTest {
Mockito.when(resourcesService.queryFileStatus(user.getUserName(), storageEntity.getFullName()))
.thenReturn(storageEntity);
StorageEntity result = pythonGateway.queryResourcesFileInfo(user.getUserName(), storageEntity.getFullName());
Assertions.assertEquals(result.getId(), storageEntity.getId());
Assertions.assertEquals(result.getFullName(), storageEntity.getFullName());
}
private StorageEntity getTestResource() {
StorageEntity storageEntity = new StorageEntity();
storageEntity.setId(1);
storageEntity.setType(ResourceType.FILE);
storageEntity.setFullName("/dev/test.py");
return storageEntity;

657
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
View File

@ -1,657 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections4.CollectionUtils;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import org.springframework.mock.web.MockMultipartFile;
import com.google.common.io.Files;
/**
* resources service test
*/
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
public class ResourcesServiceTest {
private static final String basePath = "/dolphinscheduler";
private static final String tenantCode = "123";
private static final String tenantFileResourceDir = "/dolphinscheduler/123/resources/";
private static final String tenantUdfResourceDir = "/dolphinscheduler/123/udfs/";
@InjectMocks
private ResourcesServiceImpl resourcesService;
@Mock
private TenantMapper tenantMapper;
@Mock
private StorageOperate storageOperate;
@Mock
private UserMapper userMapper;
@Mock
private UdfFuncMapper udfFunctionMapper;
@Mock
private ProcessDefinitionMapper processDefinitionMapper;
@Mock
private ResourcePermissionCheckService resourcePermissionCheckService;
private MockedStatic<FileUtils> mockedStaticFileUtils;
private MockedStatic<Files> mockedStaticFiles;
private MockedStatic<org.apache.dolphinscheduler.api.utils.FileUtils> mockedStaticDolphinschedulerFileUtils;
private MockedStatic<PropertyUtils> mockedStaticPropertyUtils;
private MockedStatic<Paths> mockedStaticPaths;
private MockedStatic<java.nio.file.Files> filesMockedStatic;
private Exception exception;
@BeforeEach
public void setUp() {
mockedStaticFileUtils = Mockito.mockStatic(FileUtils.class);
mockedStaticFiles = Mockito.mockStatic(Files.class);
mockedStaticDolphinschedulerFileUtils =
Mockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class);
mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
mockedStaticPaths = Mockito.mockStatic(Paths.class);
filesMockedStatic = Mockito.mockStatic(java.nio.file.Files.class);
}
@AfterEach
public void after() {
mockedStaticFileUtils.close();
mockedStaticFiles.close();
mockedStaticDolphinschedulerFileUtils.close();
mockedStaticPropertyUtils.close();
mockedStaticPaths.close();
filesMockedStatic.close();
}
@Test
public void testCreateResource() {
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
// CURRENT_LOGIN_USER_TENANT_NOT_EXIST
when(userMapper.selectById(user.getId())).thenReturn(getUser());
when(tenantMapper.queryById(1)).thenReturn(null);
ServiceException serviceException = Assertions.assertThrows(ServiceException.class,
() -> resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()), "/"));
assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), serviceException.getMessage());
// set tenant for user
user.setTenantId(1);
when(tenantMapper.queryById(1)).thenReturn(getTenant());
when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
// ILLEGAL_RESOURCE_FILE
String illegal_path = "/dolphinscheduler/123/../";
serviceException = Assertions.assertThrows(ServiceException.class,
() -> {
MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
mockMultipartFile, illegal_path);
});
assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, illegal_path), serviceException);
// RESOURCE_FILE_IS_EMPTY
MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
Result result = resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
mockMultipartFile, tenantFileResourceDir);
assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg());
// RESOURCE_SUFFIX_FORBID_CHANGE
mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
when(Files.getFileExtension("test.pdf")).thenReturn("pdf");
when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.uploadResource(user, "ResourcesServiceTest.jar", ResourceType.FILE, mockMultipartFile,
tenantFileResourceDir);
assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
// UDF_RESOURCE_SUFFIX_NOT_JAR
mockMultipartFile =
new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes());
when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf");
result = resourcesService.uploadResource(user, "ResourcesServiceTest.pdf", ResourceType.UDF, mockMultipartFile,
tenantUdfResourceDir);
assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
// FULL_FILE_NAME_TOO_LONG
String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf";
mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes());
when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf");
// '/databasePath/tenantCode/RESOURCE/'
when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
result = resourcesService.uploadResource(user, tooLongFileName, ResourceType.FILE, mockMultipartFile,
tenantFileResourceDir);
assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg());
}
@Test
public void testCreateDirecotry() throws IOException {
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
String fileName = "directoryTest";
// RESOURCE_EXIST
user.setId(1);
user.setTenantId(1);
when(tenantMapper.queryById(1)).thenReturn(getTenant());
when(userMapper.selectById(user.getId())).thenReturn(getUser());
when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true);
Result result = resourcesService.createDirectory(user, fileName, ResourceType.FILE, -1, tenantFileResourceDir);
assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
}
@Test
public void testUpdateResource() throws Exception {
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
user.setTenantId(1);
when(userMapper.selectById(user.getId())).thenReturn(getUser());
when(tenantMapper.queryById(1)).thenReturn(getTenant());
when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
// TENANT_NOT_EXIST
when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null);
Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResource(user,
"ResourcesServiceTest1.jar", "", "ResourcesServiceTest", ResourceType.UDF, null));
// USER_NO_OPERATION_PERM
user.setUserType(UserType.GENERAL_USER);
// tenant who have access to resource is 123,
Tenant tenantWNoPermission = new Tenant();
tenantWNoPermission.setTenantCode("321");
when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission);
when(storageOperate.getDir(ResourceType.ALL, "321")).thenReturn(basePath);
String fileName = "ResourcesServiceTest";
Result result = resourcesService.updateResource(user, tenantFileResourceDir + fileName,
tenantCode, fileName, ResourceType.FILE, null);
assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg());
// SUCCESS
when(tenantMapper.queryById(1)).thenReturn(getTenant());
when(storageOperate.exists(Mockito.any())).thenReturn(false);
when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir);
when(storageOperate.getFileStatus(tenantFileResourceDir + fileName,
tenantFileResourceDir, tenantCode, ResourceType.FILE))
.thenReturn(getStorageEntityResource(fileName));
result = resourcesService.updateResource(user, tenantFileResourceDir + fileName,
tenantCode, fileName, ResourceType.FILE, null);
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
// Tests for udf resources.
fileName = "ResourcesServiceTest.jar";
when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir);
when(storageOperate.exists(tenantUdfResourceDir + fileName)).thenReturn(true);
when(storageOperate.getFileStatus(tenantUdfResourceDir + fileName, tenantUdfResourceDir, tenantCode,
ResourceType.UDF))
.thenReturn(getStorageEntityUdfResource(fileName));
result = resourcesService.updateResource(user, tenantUdfResourceDir + fileName,
tenantCode, fileName, ResourceType.UDF, null);
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testQueryResourceListPaging() throws Exception {
User loginUser = new User();
loginUser.setId(1);
loginUser.setTenantId(1);
loginUser.setTenantCode("tenant1");
loginUser.setUserType(UserType.ADMIN_USER);
String fileName = "ResourcesServiceTest";
List<StorageEntity> mockResList = new ArrayList<>();
mockResList.add(getStorageEntityResource(fileName));
List<User> mockUserList = new ArrayList<>();
mockUserList.add(getUser());
when(userMapper.selectList(null)).thenReturn(mockUserList);
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
when(storageOperate.listFilesStatus(tenantFileResourceDir, tenantFileResourceDir,
tenantCode, ResourceType.FILE)).thenReturn(mockResList);
Result result = resourcesService.queryResourceListPaging(loginUser, "", "", ResourceType.FILE, "Test", 1, 10);
assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
PageInfo pageInfo = (PageInfo) result.getData();
Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList()));
}
@Test
public void testQueryResourceList() {
User loginUser = getUser();
String fileName = "ResourcesServiceTest";
when(userMapper.selectList(null)).thenReturn(Collections.singletonList(loginUser));
when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir);
when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir);
when(storageOperate.listFilesStatusRecursively(tenantFileResourceDir,
tenantFileResourceDir, tenantCode, ResourceType.FILE))
.thenReturn(Collections.singletonList(getStorageEntityResource(fileName)));
Map<String, Object> result =
resourcesService.queryResourceList(loginUser, ResourceType.FILE, tenantFileResourceDir);
assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<ResourceComponent> resourceList = (List<ResourceComponent>) result.get(Constants.DATA_LIST);
Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
// test udf
when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir);
when(storageOperate.getUdfDir(tenantCode)).thenReturn(tenantUdfResourceDir);
when(storageOperate.listFilesStatusRecursively(tenantUdfResourceDir, tenantUdfResourceDir,
tenantCode, ResourceType.UDF)).thenReturn(Arrays.asList(getStorageEntityUdfResource("test.jar")));
loginUser.setUserType(UserType.GENERAL_USER);
result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, tenantUdfResourceDir);
assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
resourceList = (List<ResourceComponent>) result.get(Constants.DATA_LIST);
Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
}
@Test
public void testDelete() throws Exception {
User loginUser = new User();
loginUser.setId(0);
loginUser.setUserType(UserType.GENERAL_USER);
// TENANT_NOT_EXIST
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setTenantId(2);
when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
Assertions.assertThrows(ServiceException.class, () -> resourcesService.delete(loginUser, "", ""));
// RESOURCE_NOT_EXIST
String fileName = "ResourcesServiceTest";
when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir);
when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, tenantFileResourceDir, tenantCode, null))
.thenReturn(getStorageEntityResource(fileName));
Result result = resourcesService.delete(loginUser, tenantFileResourceDir + "ResNotExist", tenantCode);
assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// SUCCESS
loginUser.setTenantId(1);
result = resourcesService.delete(loginUser, tenantFileResourceDir + fileName, tenantCode);
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testVerifyResourceName() throws IOException {
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
String fileName = "ResourcesServiceTest";
when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true);
Result result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user);
assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
// RESOURCE_FILE_EXIST
result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user);
Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode());
// SUCCESS
result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user);
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testReadResource() throws IOException {
// RESOURCE_NOT_EXIST
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
Result result = resourcesService.readResource(getUser(), "", "", 1, 10);
assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
result = resourcesService.readResource(getUser(), "", "", 1, 10);
assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
// USER_NOT_EXIST
when(userMapper.selectById(getUser().getId())).thenReturn(null);
when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.readResource(getUser(), "", "", 1, 10);
assertEquals(Status.USER_NOT_EXIST.getCode(), (int) result.getCode());
// TENANT_NOT_EXIST
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(null);
Assertions.assertThrows(ServiceException.class, () -> resourcesService.readResource(getUser(), "", "", 1, 10));
// SUCCESS
when(FileUtils.getResourceViewSuffixes()).thenReturn("jar,sh");
when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir);
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
when(storageOperate.exists(Mockito.any())).thenReturn(true);
when(storageOperate.vimFile(Mockito.any(), Mockito.any(), eq(1), eq(10))).thenReturn(getContent());
when(Files.getFileExtension("/dolphinscheduler/123/resources/test.jar")).thenReturn("jar");
result = resourcesService.readResource(getUser(), "/dolphinscheduler/123/resources/test.jar", tenantCode, 1,
10);
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testCreateOrUpdateResource() throws Exception {
User user = getUser();
when(userMapper.queryByUserNameAccurately(user.getUserName())).thenReturn(getUser());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
exception = Assertions.assertThrows(IllegalArgumentException.class,
() -> resourcesService.createOrUpdateResource(user.getUserName(), "filename", "my-content"));
Assertions.assertTrue(
exception.getMessage().contains("Not allow create or update resources without extension name"));
// SUCCESS
String fileName = "ResourcesServiceTest";
when(storageOperate.getResDir(user.getTenantCode())).thenReturn(tenantFileResourceDir);
when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any()))
.thenReturn(getStorageEntityResource(fileName));
StorageEntity storageEntity =
resourcesService.createOrUpdateResource(user.getUserName(), "filename.txt", "my-content");
Assertions.assertNotNull(storageEntity);
assertEquals(tenantFileResourceDir + fileName, storageEntity.getFullName());
}
@Test
public void testUpdateResourceContent() throws Exception {
// RESOURCE_PATH_ILLEGAL
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
when(tenantMapper.queryById(1)).thenReturn(getTenant());
when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/tmp");
String fileName = "ResourcesServiceTest.jar";
ServiceException serviceException =
Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(),
tenantFileResourceDir + fileName, tenantCode, "content"));
assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, tenantFileResourceDir + fileName),
serviceException);
// RESOURCE_NOT_EXIST
when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath);
when(storageOperate.getResDir(Mockito.anyString())).thenReturn(tenantFileResourceDir);
when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode, ResourceType.FILE))
.thenReturn(null);
Result result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + fileName, tenantCode,
"content");
assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
when(storageOperate.getFileStatus(tenantFileResourceDir, "", tenantCode, ResourceType.FILE))
.thenReturn(getStorageEntityResource(fileName));
result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir, tenantCode,
"content");
assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
// USER_NOT_EXIST
when(userMapper.selectById(getUser().getId())).thenReturn(null);
result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + "123.class",
tenantCode,
"content");
Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
// TENANT_NOT_EXIST
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
when(tenantMapper.queryById(1)).thenReturn(null);
Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(),
tenantFileResourceDir + fileName, tenantCode, "content"));
// SUCCESS
when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode,
ResourceType.FILE)).thenReturn(getStorageEntityResource(fileName));
when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar");
when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
when(tenantMapper.queryById(1)).thenReturn(getTenant());
when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
result = resourcesService.updateResourceContent(getUser(),
tenantFileResourceDir + fileName, tenantCode, "content");
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testDownloadResource() throws IOException {
when(tenantMapper.queryById(1)).thenReturn(getTenant());
when(userMapper.selectById(1)).thenReturn(getUser());
org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class);
Path path = Mockito.mock(Path.class);
when(Paths.get(Mockito.any())).thenReturn(path);
when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L);
// resource null
org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNull(resource);
when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock);
resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNotNull(resource);
}
@Test
public void testDeleteDataTransferData() throws Exception {
User user = getUser();
when(userMapper.selectById(user.getId())).thenReturn(getUser());
when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
StorageEntity storageEntity1 = Mockito.mock(StorageEntity.class);
StorageEntity storageEntity2 = Mockito.mock(StorageEntity.class);
StorageEntity storageEntity3 = Mockito.mock(StorageEntity.class);
StorageEntity storageEntity4 = Mockito.mock(StorageEntity.class);
StorageEntity storageEntity5 = Mockito.mock(StorageEntity.class);
when(storageEntity1.getFullName()).thenReturn("DATA_TRANSFER/20220101");
when(storageEntity2.getFullName()).thenReturn("DATA_TRANSFER/20220102");
when(storageEntity3.getFullName()).thenReturn("DATA_TRANSFER/20220103");
when(storageEntity4.getFullName()).thenReturn("DATA_TRANSFER/20220104");
when(storageEntity5.getFullName()).thenReturn("DATA_TRANSFER/20220105");
List<StorageEntity> storageEntityList = new ArrayList<>();
storageEntityList.add(storageEntity1);
storageEntityList.add(storageEntity2);
storageEntityList.add(storageEntity3);
storageEntityList.add(storageEntity4);
storageEntityList.add(storageEntity5);
when(storageOperate.listFilesStatus(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
.thenReturn(storageEntityList);
LocalDateTime localDateTime = LocalDateTime.of(2022, 1, 5, 0, 0, 0);
try (MockedStatic<LocalDateTime> mockHook = Mockito.mockStatic(LocalDateTime.class)) {
mockHook.when(LocalDateTime::now).thenReturn(localDateTime);
DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 3);
assertEquals(response.getSuccessList().size(), 2);
assertEquals(response.getSuccessList().get(0), "DATA_TRANSFER/20220101");
assertEquals(response.getSuccessList().get(1), "DATA_TRANSFER/20220102");
}
try (MockedStatic<LocalDateTime> mockHook = Mockito.mockStatic(LocalDateTime.class)) {
mockHook.when(LocalDateTime::now).thenReturn(localDateTime);
DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 0);
assertEquals(response.getSuccessList().size(), 5);
}
}
@Test
public void testCatFile() throws IOException {
// SUCCESS
List<String> list = storageOperate.vimFile(Mockito.any(), Mockito.anyString(), eq(1), eq(10));
Assertions.assertNotNull(list);
}
@Test
void testQueryBaseDir() throws Exception {
User user = getUser();
String fileName = "ResourcesServiceTest.jar";
when(userMapper.selectById(user.getId())).thenReturn(getUser());
when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir);
when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
Mockito.any())).thenReturn(getStorageEntityResource(fileName));
Result<Object> result = resourcesService.queryResourceBaseDir(user, ResourceType.FILE);
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
private Tenant getTenant() {
Tenant tenant = new Tenant();
tenant.setTenantCode(tenantCode);
return tenant;
}
private User getUser() {
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
user.setTenantId(1);
user.setTenantCode(tenantCode);
return user;
}
private StorageEntity getStorageEntityResource(String fileName) {
StorageEntity entity = new StorageEntity();
entity.setAlias(fileName);
entity.setFileName(fileName);
entity.setDirectory(false);
entity.setUserName(tenantCode);
entity.setType(ResourceType.FILE);
entity.setFullName(tenantFileResourceDir + fileName);
return entity;
}
private StorageEntity getStorageEntityUdfResource(String fileName) {
StorageEntity entity = new StorageEntity();
entity.setAlias(fileName);
entity.setFileName(fileName);
entity.setDirectory(false);
entity.setUserName(tenantCode);
entity.setType(ResourceType.UDF);
entity.setFullName(tenantUdfResourceDir + fileName);
return entity;
}
private List<String> getContent() {
List<String> contentList = new ArrayList<>();
contentList.add("test");
return contentList;
}
private List<Map<String, Object>> getResources() {
List<Map<String, Object>> resources = new ArrayList<>();
Map<String, Object> resource = new HashMap<>();
resource.put("id", 1);
resource.put("resource_ids", "1");
resources.add(resource);
return resources;
}
private static String getRandomStringWithLength(int length) {
Random r = new Random();
StringBuilder sb = new StringBuilder();
while (sb.length() < length) {
char c = (char) (r.nextInt(26) + 'a');
sb.append(c);
}
return sb.toString();
}
}

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
View File

@ -41,7 +41,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
@ -95,7 +95,7 @@ public class TenantServiceTest {
private ResourcePermissionCheckService resourcePermissionCheckService;
@Mock
private StorageOperate storageOperate;
private StorageOperator storageOperator;
private static final String tenantCode = "hayden";
private static final String tenantDesc = "This is the tenant desc";

17
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
View File

@ -32,11 +32,10 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -83,7 +82,7 @@ public class UdfFuncServiceTest {
private UDFUserMapper udfUserMapper;
@Mock
private StorageOperate storageOperate;
private StorageOperator storageOperator;
@BeforeEach
public void setUp() {
@ -111,11 +110,7 @@ public class UdfFuncServiceTest {
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// success
try {
Mockito.when(storageOperate.exists("String")).thenReturn(true);
} catch (IOException e) {
logger.error("AmazonServiceException when checking resource: String");
}
Mockito.when(storageOperator.exists("String")).thenReturn(true);
result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
@ -176,11 +171,7 @@ public class UdfFuncServiceTest {
// success
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, 1,
ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE, serviceLogger)).thenReturn(true);
try {
Mockito.when(storageOperate.exists("")).thenReturn(true);
} catch (IOException e) {
logger.error("AmazonServiceException when checking resource: ");
}
Mockito.when(storageOperator.exists("")).thenReturn(true);
result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
View File

@ -48,7 +48,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
@ -117,7 +117,7 @@ public class UsersServiceTest {
private ProjectMapper projectMapper;
@Mock
private StorageOperate storageOperate;
private StorageOperator storageOperator;
@Mock
private ResourcePermissionCheckService resourcePermissionCheckService;

136
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java Normal file
View File

@ -0,0 +1,136 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import java.util.Locale;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.i18n.LocaleContextHolder;
@ExtendWith(MockitoExtension.class)
class CreateDirectoryDtoValidatorTest {
@Mock
private StorageOperator storageOperator;
@Mock
private TenantDao tenantDao;
@InjectMocks
private CreateDirectoryDtoValidator createDirectoryDtoValidator;
private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
private User loginUser;
@BeforeEach
public void setup() {
when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
loginUser = new User();
loginUser.setTenantId(1);
LocaleContextHolder.setLocale(Locale.ENGLISH);
}
@Test
void testValidate_notUnderBaseDirectory() {
CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
.loginUser(loginUser)
.directoryAbsolutePath("/tmp")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp",
() -> createDirectoryDtoValidator.validate(createDirectoryDto));
}
@Test
public void testValidate_directoryPathContainsIllegalSymbolic() {
CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
.loginUser(loginUser)
.directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
() -> createDirectoryDtoValidator.validate(createDirectoryDto));
}
@Test
public void testValidate_directoryExist() {
CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
.loginUser(loginUser)
.directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo")
.build();
when(storageOperator.exists(createDirectoryDto.getDirectoryAbsolutePath())).thenReturn(true);
assertThrowServiceException(
"Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/demo",
() -> createDirectoryDtoValidator.validate(createDirectoryDto));
}
@Test
public void testValidate_NoPermission() {
Tenant tenant = new Tenant();
tenant.setTenantCode("test");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
.loginUser(loginUser)
.directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo")
.build();
when(storageOperator.getResourceMetaData(createDirectoryDto.getDirectoryAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(createDirectoryDto.getDirectoryAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("demo")
.isDirectory(true)
.tenant("default")
.build());
when(storageOperator.exists(createDirectoryDto.getDirectoryAbsolutePath())).thenReturn(false);
assertThrowServiceException(
"Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/demo",
() -> createDirectoryDtoValidator.validate(createDirectoryDto));
}
@Test
public void testValidate_pathNotDirectory() {
CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder()
.loginUser(loginUser)
.directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo.sql")
.build();
loginUser.setUserType(UserType.ADMIN_USER);
assertThrowServiceException(
"Internal Server Error: The path is not a directory: /tmp/dolphinscheduler/default/resources/demo.sql",
() -> createDirectoryDtoValidator.validate(createDirectoryDto));
}
}

188
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java Normal file
View File

@ -0,0 +1,188 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow;
import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import java.util.Locale;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.i18n.LocaleContextHolder;
@ExtendWith(MockitoExtension.class)
class CreateFileFromContentDtoValidatorTest {
@Mock
private StorageOperator storageOperator;
@Mock
private TenantDao tenantDao;
@InjectMocks
private CreateFileFromContentDtoValidator createFileFromContentDtoValidator;
private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
private User loginUser;
@BeforeEach
public void setup() {
when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
loginUser = new User();
loginUser.setTenantId(1);
LocaleContextHolder.setLocale(Locale.ENGLISH);
}
@Test
void testValidate_notUnderBaseDirectory() {
CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
.loginUser(loginUser)
.fileAbsolutePath("/tmp")
.fileContent("select * from t")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp",
() -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
}
@Test
public void testValidate_filePathContainsIllegalSymbolic() {
CreateFileFromContentDto renameDirectoryDto = CreateFileFromContentDto.builder()
.loginUser(loginUser)
.fileAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
.fileContent("select * from t")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
() -> createFileFromContentDtoValidator.validate(renameDirectoryDto));
}
@Test
public void testValidate_IsNotFile() {
CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
.loginUser(loginUser)
.fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.fileContent("select * from t")
.build();
assertThrowServiceException(
"Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a",
() -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
}
@Test
public void testValidate_fileAlreadyExist() {
CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
.loginUser(loginUser)
.fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
.fileContent("select * from t")
.build();
when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(true);
assertThrowServiceException(
"Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/a.sql",
() -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
}
@Test
public void testValidate_fileNoPermission() {
Tenant tenant = new Tenant();
tenant.setTenantCode("test");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
.loginUser(loginUser)
.fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
.fileContent("select * from t")
.build();
when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.sql")
.isDirectory(false)
.tenant("default")
.build());
assertThrowServiceException(
"Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.sql",
() -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
}
@Test
public void testValidate_contentIsInvalidated() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
.loginUser(loginUser)
.fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
.fileContent("")
.build();
when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.sql")
.isDirectory(false)
.tenant("default")
.build());
assertThrowServiceException(
"Internal Server Error: The file content is null",
() -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
}
@Test
public void testValidate() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder()
.loginUser(loginUser)
.fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
.fileContent("select * from t")
.build();
when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.sql")
.isDirectory(false)
.tenant("default")
.build());
assertDoesNotThrow(() -> createFileFromContentDtoValidator.validate(createFileFromContentDto));
}
}

197
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java Normal file
View File

@ -0,0 +1,197 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.api.AssertionsHelper;
import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import java.util.Locale;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import org.springframework.context.i18n.LocaleContextHolder;
@MockitoSettings(strictness = Strictness.LENIENT)
@ExtendWith(MockitoExtension.class)
class FetchFileContentDtoValidatorTest {
@Mock
private StorageOperator storageOperator;
@Mock
private TenantDao tenantDao;
@InjectMocks
private FetchFileContentDtoValidator fetchFileContentDtoValidator;
private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
private User loginUser;
@BeforeEach
public void setup() {
when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
loginUser = new User();
loginUser.setTenantId(1);
LocaleContextHolder.setLocale(Locale.ENGLISH);
}
@Test
void testValidate_skipLineNumInvalid() {
FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath("/tmp")
.skipLineNum(-1)
.limit(-1)
.build();
assertThrowServiceException(
"Internal Server Error: skipLineNum must be greater than or equal to 0",
() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
}
@Test
void testValidate_notUnderBaseDirectory() {
FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath("/tmp")
.skipLineNum(0)
.limit(-1)
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp",
() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
}
@Test
public void testValidate_filePathContainsIllegalSymbolic() {
FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
.skipLineNum(0)
.limit(-1)
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
}
@Test
public void testValidate_IsNotFile() {
FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.skipLineNum(0)
.limit(-1)
.build();
assertThrowServiceException(
"Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a",
() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
}
@Test
public void testValidate_fileNoPermission() {
Tenant tenant = new Tenant();
tenant.setTenantCode("test");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
.skipLineNum(0)
.limit(-1)
.build();
when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.sql")
.isDirectory(false)
.tenant("default")
.build());
assertThrowServiceException(
"Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.sql",
() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
}
@Test
void validate_fileExtensionInvalid() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.jar")
.skipLineNum(0)
.limit(-1)
.build();
when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.jar")
.isDirectory(false)
.tenant("default")
.build());
assertThrowServiceException(
"Internal Server Error: The file type: jar cannot be fetched",
() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
}
@Test
void validate() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql")
.skipLineNum(0)
.limit(-1)
.build();
when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.sql")
.isDirectory(false)
.tenant("default")
.build());
AssertionsHelper.assertDoesNotThrow(() -> fetchFileContentDtoValidator.validate(fetchFileContentDto));
}
}

188
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java Normal file
View File

@ -0,0 +1,188 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow;
import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import java.util.Locale;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.i18n.LocaleContextHolder;
@ExtendWith(MockitoExtension.class)
class RenameDirectoryDtoValidatorTest {
@Mock
private StorageOperator storageOperator;
@Mock
private TenantDao tenantDao;
@InjectMocks
private RenameDirectoryDtoValidator renameDirectoryDtoValidator;
private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
private User loginUser;
@BeforeEach
public void setup() {
when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
loginUser = new User();
loginUser.setTenantId(1);
LocaleContextHolder.setLocale(Locale.ENGLISH);
}
@Test
void testValidate_notUnderBaseDirectory() {
RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
.loginUser(loginUser)
.originDirectoryAbsolutePath("/tmp")
.targetDirectoryAbsolutePath("/tmp1")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp",
() -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
}
@Test
public void testValidate_directoryPathContainsIllegalSymbolic() {
RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
.loginUser(loginUser)
.originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/..")
.targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..",
() -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
}
@Test
public void testValidate_originDirectoryNotExist() {
RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
.loginUser(loginUser)
.originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
.build();
assertThrowServiceException(
"Internal Server Error: Thr resource is not exists: /tmp/dolphinscheduler/default/resources/a",
() -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
}
@Test
public void testValidate_originDirectoryNoPermission() {
Tenant tenant = new Tenant();
tenant.setTenantCode("test");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
.loginUser(loginUser)
.originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
.build();
when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true);
when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a")
.isDirectory(true)
.tenant("default")
.build());
assertThrowServiceException(
"Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a",
() -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
}
@Test
public void testValidate_targetDirectoryAlreadyExist() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
.loginUser(loginUser)
.originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
.build();
when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true);
when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a")
.isDirectory(true)
.tenant("default")
.build());
when(storageOperator.exists(renameDirectoryDto.getTargetDirectoryAbsolutePath())).thenReturn(true);
assertThrowServiceException(
"Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/b",
() -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
}
@Test
public void testValidate() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder()
.loginUser(loginUser)
.originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b")
.build();
when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true);
when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a")
.isDirectory(true)
.tenant("default")
.build());
when(storageOperator.exists(renameDirectoryDto.getTargetDirectoryAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(renameDirectoryDto.getTargetDirectoryAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("b")
.isDirectory(true)
.tenant("default")
.build());
assertDoesNotThrow(() -> renameDirectoryDtoValidator.validate(renameDirectoryDto));
}
}

202
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java Normal file
View File

@ -0,0 +1,202 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import java.util.Locale;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.context.i18n.LocaleContextHolder;
@ExtendWith(MockitoExtension.class)
public class RenameFileDtoValidatorTest {
@Mock
private StorageOperator storageOperator;
@Mock
private TenantDao tenantDao;
@InjectMocks
private RenameFileDtoValidator renameFileDtoValidator;
private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler";
private User loginUser;
@BeforeEach
public void setup() {
when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY);
loginUser = new User();
loginUser.setTenantId(1);
LocaleContextHolder.setLocale(Locale.ENGLISH);
}
@Test
void testValidate_notUnderBaseDirectory() {
RenameFileDto renameFileDto = RenameFileDto.builder()
.loginUser(loginUser)
.originFileAbsolutePath("/tmp")
.targetFileAbsolutePath("/tmp1")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp",
() -> renameFileDtoValidator.validate(renameFileDto));
}
@Test
public void testValidate_fileAbsolutePathContainsIllegalSymbolic() {
RenameFileDto renameFileDto = RenameFileDto.builder()
.loginUser(loginUser)
.originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/../a.txt")
.targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
.build();
assertThrowServiceException(
"Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/../a.txt",
() -> renameFileDtoValidator.validate(renameFileDto));
}
@Test
public void testValidate_originFileNotExist() {
RenameFileDto renameFileDto = RenameFileDto.builder()
.loginUser(loginUser)
.originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
.targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
.build();
assertThrowServiceException(
"Internal Server Error: Thr resource is not exists: /tmp/dolphinscheduler/default/resources/a.txt",
() -> renameFileDtoValidator.validate(renameFileDto));
}
@Test
public void testValidate_originFileIsNotFile() {
RenameFileDto renameFileDto = RenameFileDto.builder()
.loginUser(loginUser)
.originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a")
.targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
.build();
when(storageOperator.exists(renameFileDto.getOriginFileAbsolutePath())).thenReturn(true);
assertThrowServiceException(
"Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a",
() -> renameFileDtoValidator.validate(renameFileDto));
}
@Test
public void testValidate_originFileNoPermission() {
Tenant tenant = new Tenant();
tenant.setTenantCode("test");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
RenameFileDto renameFileDto = RenameFileDto.builder()
.loginUser(loginUser)
.originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
.targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
.build();
when(storageOperator.exists(renameFileDto.getOriginFileAbsolutePath())).thenReturn(true);
when(storageOperator.getResourceMetaData(renameFileDto.getOriginFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameFileDto.getOriginFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.txt")
.isDirectory(false)
.tenant("default")
.build());
assertThrowServiceException(
"Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.txt",
() -> renameFileDtoValidator.validate(renameFileDto));
}
@Test
public void testValidate_targetFileAlreadyExist() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
RenameFileDto renameDirectoryDto = RenameFileDto.builder()
.loginUser(loginUser)
.originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
.targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
.build();
when(storageOperator.exists(renameDirectoryDto.getOriginFileAbsolutePath())).thenReturn(true);
when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameDirectoryDto.getOriginFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.txt")
.isDirectory(false)
.tenant("default")
.build());
when(storageOperator.exists(renameDirectoryDto.getTargetFileAbsolutePath())).thenReturn(true);
assertThrowServiceException(
"Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/b.txt",
() -> renameFileDtoValidator.validate(renameDirectoryDto));
}
@Test
public void testValidate() {
Tenant tenant = new Tenant();
tenant.setTenantCode("default");
when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant));
RenameFileDto renameDirectoryDto = RenameFileDto.builder()
.loginUser(loginUser)
.originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt")
.targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt")
.build();
when(storageOperator.exists(renameDirectoryDto.getOriginFileAbsolutePath())).thenReturn(true);
when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameDirectoryDto.getOriginFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("a.txt")
.isDirectory(false)
.tenant("default")
.build());
when(storageOperator.exists(renameDirectoryDto.getTargetFileAbsolutePath())).thenReturn(false);
when(storageOperator.getResourceMetaData(renameDirectoryDto.getTargetFileAbsolutePath()))
.thenReturn(ResourceMetadata.builder()
.resourceAbsolutePath(renameDirectoryDto.getTargetFileAbsolutePath())
.resourceBaseDirectory(BASE_DIRECTORY)
.resourceRelativePath("b.txt")
.isDirectory(false)
.tenant("default")
.build());
assertDoesNotThrow(() -> renameFileDtoValidator.validate(renameDirectoryDto));
}
}

7
dolphinscheduler-bom/pom.xml
View File

@ -965,6 +965,13 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>minio</artifactId>
<version>${testcontainer.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.checkerframework</groupId>
<artifactId>checker-qual</artifactId>

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java
View File

@ -43,6 +43,10 @@ public class ImmutableYamlDelegate implements IPropertyDelegate {
// read from classpath
for (String fileName : yamlAbsolutePath) {
try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) {
if (fis == null) {
log.warn("Cannot find the file: {} under classpath", fileName);
continue;
}
YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
factory.setResources(new InputStreamResource(fis));
factory.afterPropertiesSet();

24
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
View File

@ -59,28 +59,6 @@ public final class Constants {
*/
public static final String HDFS_DEFAULT_FS = "fs.defaultFS";
/**
* hadoop configuration
*/
public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE";
public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port";
/**
* yarn.resourcemanager.ha.rm.ids
*/
public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids";
/**
* yarn.application.status.address
*/
public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address";
/**
* yarn.job.history.status.address
*/
public static final String YARN_JOB_HISTORY_STATUS_ADDRESS = "yarn.job.history.status.address";
/**
* hdfs configuration
* resource.hdfs.root.user
@ -392,8 +370,6 @@ public final class Constants {
public static final String QUEUE_NAME = "queueName";
public static final int LOG_QUERY_SKIP_LINE_NUMBER = 0;
public static final int LOG_QUERY_LIMIT = 4096;
public static final String ALIAS = "alias";
public static final String CONTENT = "content";
public static final String DEPENDENT_SPLIT = ":||";
public static final long DEPENDENT_ALL_TASK_CODE = -1;
public static final long DEPENDENT_WORKFLOW_CODE = 0;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StorageType.java
View File

@ -20,6 +20,6 @@ package org.apache.dolphinscheduler.common.enums;
/**
* data base types
*/
public enum ResUploadType {
public enum StorageType {
LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
}

75
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
View File

@ -17,14 +17,15 @@
package org.apache.dolphinscheduler.common.utils;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.dolphinscheduler.common.constants.Constants.DATA_BASEDIR_PATH;
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import java.io.ByteArrayOutputStream;
@ -33,17 +34,21 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Optional;
import java.util.Set;
import java.util.zip.CRC32;
import java.util.zip.CheckedInputStream;
import lombok.NonNull;
import lombok.SneakyThrows;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@ -66,32 +71,14 @@ public class FileUtils {
* @return download file name
*/
public static String getDownloadFilename(String filename) {
String fileName =
String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename);
File file = new File(fileName);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
return fileName;
return Paths.get(DATA_BASEDIR, "tmp", CodeGenerateUtils.genCode() + "-" + filename).toString();
}
/**
* get upload file absolute path and name
*
* @param tenantCode tenant code
* @param filename file name
* @return local file path
* Generate a local tmp absolute path of the uploaded file
*/
public static String getUploadFilename(String tenantCode, String filename) {
String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename);
File file = new File(fileName);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
return fileName;
public static String getUploadFileLocalTmpAbsolutePath() {
return Paths.get(DATA_BASEDIR, "tmp", String.valueOf(CodeGenerateUtils.genCode())).toString();
}
/**
@ -236,6 +223,7 @@ public class FileUtils {
/**
* Calculate file checksum with CRC32 algorithm
*
* @param pathName
* @return checksum of file/dir
*/
@ -315,4 +303,45 @@ public class FileUtils {
}
}
public static String concatFilePath(String... paths) {
if (paths.length == 0) {
throw new IllegalArgumentException("At least one path should be provided");
}
StringBuilder finalPath = new StringBuilder(paths[0]);
if (StringUtils.isEmpty(finalPath)) {
throw new IllegalArgumentException("The path should not be empty");
}
String separator = File.separator;
for (int i = 1; i < paths.length; i++) {
String path = paths[i];
if (StringUtils.isEmpty(path)) {
throw new IllegalArgumentException("The path should not be empty");
}
if (finalPath.toString().endsWith(separator) && path.startsWith(separator)) {
finalPath.append(path.substring(separator.length()));
continue;
}
if (!finalPath.toString().endsWith(separator) && !path.startsWith(separator)) {
finalPath.append(separator).append(path);
continue;
}
finalPath.append(path);
}
return finalPath.toString();
}
public static String getClassPathAbsolutePath(Class clazz) {
checkNotNull(clazz, "class is null");
return Optional.ofNullable(clazz.getResource("/"))
.map(URL::getPath)
.orElseThrow(() -> new IllegalArgumentException("class path: " + clazz + " is null"));
}
/**
* copy input stream to file, if the file already exists, will append the content to the beginning of the file, otherwise will create a new file.
*/
@SneakyThrows
public static void copyInputStreamToFile(InputStream inputStream, String destFilename) {
org.apache.commons.io.FileUtils.copyInputStreamToFile(inputStream, new File(destFilename));
}
}

15
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
View File

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@ -31,26 +29,21 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.google.common.truth.Truth;
@ExtendWith(MockitoExtension.class)
public class FileUtilsTest {
@Test
public void testGetDownloadFilename() {
try (MockedStatic<DateUtils> mockedDateUtils = Mockito.mockStatic(DateUtils.class)) {
mockedDateUtils.when(() -> DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059");
Assertions.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test",
FileUtils.getDownloadFilename("test"));
}
Truth.assertThat(FileUtils.getDownloadFilename("test")).startsWith("/tmp/dolphinscheduler/tmp/");
}
@Test
public void testGetUploadFilename() {
Assertions.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb",
FileUtils.getUploadFilename("aaa", "bbb"));
Truth.assertThat(FileUtils.getUploadFileLocalTmpAbsolutePath()).startsWith("/tmp/dolphinscheduler/tmp/");
}
@Test

24
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.repository;
import org.apache.dolphinscheduler.dao.entity.Tenant;
public interface TenantDao extends IDao<Tenant> {
}

36
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.repository.impl;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.repository.BaseDao;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import lombok.NonNull;
import org.springframework.stereotype.Repository;
@Repository
public class TenantDaoImpl extends BaseDao<Tenant, TenantMapper> implements TenantDao {
public TenantDaoImpl(@NonNull TenantMapper tenantMapper) {
super(tenantMapper);
}
}

22
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java
View File

@ -22,7 +22,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.CRC_SUFFIX;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
@ -65,17 +65,17 @@ public class TaskCacheUtils {
* 4. input VarPool, from upstream task and workflow global parameters
* @param taskInstance task instance
* @param taskExecutionContext taskExecutionContext
* @param storageOperate storageOperate
* @param storageOperator storageOperate
* @return cache key
*/
public static String generateCacheKey(TaskInstance taskInstance, TaskExecutionContext taskExecutionContext,
StorageOperate storageOperate) {
StorageOperator storageOperator) {
List<String> keyElements = new ArrayList<>();
keyElements.add(String.valueOf(taskInstance.getTaskCode()));
keyElements.add(String.valueOf(taskInstance.getTaskDefinitionVersion()));
keyElements.add(String.valueOf(taskInstance.getIsCache().getCode()));
keyElements.add(String.valueOf(taskInstance.getEnvironmentConfig()));
keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate));
keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator));
String data = StringUtils.join(keyElements, "_");
return DigestUtils.sha256Hex(data);
}
@ -123,7 +123,7 @@ public class TaskCacheUtils {
* taskExecutionContext taskExecutionContext
*/
public static String getTaskInputVarPoolData(TaskInstance taskInstance, TaskExecutionContext context,
StorageOperate storageOperate) {
StorageOperator storageOperator) {
JsonNode taskParams = JSONUtils.parseObject(taskInstance.getTaskParams());
// The set of input values considered from localParams in the taskParams
@ -141,7 +141,8 @@ public class TaskCacheUtils {
List<Property> fileInput = varPool.stream().filter(property -> property.getType().equals(DataType.FILE))
.collect(Collectors.toList());
fileInput.forEach(
property -> fileCheckSumMap.put(property.getProp(), getValCheckSum(property, context, storageOperate)));
property -> fileCheckSumMap.put(property.getProp(),
getValCheckSum(property, context, storageOperator)));
// var pool value from workflow global parameters
if (context.getPrepareParamsMap() != null) {
@ -173,17 +174,18 @@ public class TaskCacheUtils {
* cache can be used if content of upstream output files are the same
* @param fileProperty
* @param context
* @param storageOperate
* @param storageOperator
*/
public static String getValCheckSum(Property fileProperty, TaskExecutionContext context,
StorageOperate storageOperate) {
StorageOperator storageOperator) {
String resourceCRCPath = fileProperty.getValue() + CRC_SUFFIX;
String resourceCRCWholePath = storageOperate.getResourceFullName(context.getTenantCode(), resourceCRCPath);
String resourceCRCWholePath =
storageOperator.getStorageFileAbsolutePath(context.getTenantCode(), resourceCRCPath);
String targetPath = String.format("%s/%s", context.getExecutePath(), resourceCRCPath);
log.info("{} --- Remote:{} to Local:{}", "CRC file", resourceCRCWholePath, targetPath);
String crcString = "";
try {
storageOperate.download(resourceCRCWholePath, targetPath, true);
storageOperator.download(resourceCRCWholePath, targetPath, true);
crcString = FileUtils.readFile2Str(new FileInputStream(targetPath));
fileProperty.setValue(crcString);
} catch (IOException e) {

22
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java
View File

@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.dao.utils;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
@ -46,7 +46,7 @@ class TaskCacheUtilsTest {
private TaskExecutionContext taskExecutionContext;
private StorageOperate storageOperate;
private StorageOperator storageOperator;
@BeforeEach
void setUp() {
@ -101,7 +101,7 @@ class TaskCacheUtilsTest {
prepareParamsMap.put("a", property);
taskExecutionContext.setPrepareParamsMap(prepareParamsMap);
storageOperate = Mockito.mock(StorageOperate.class);
storageOperator = Mockito.mock(StorageOperator.class);
}
@Test
@ -128,26 +128,26 @@ class TaskCacheUtilsTest {
@Test
void TestGetTaskInputVarPoolData() {
TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate);
TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator);
// only a=aa and c=cc will influence the result,
// b=bb is a fixed value, will be considered in task version
// k=kk is not in task params, will be ignored
String except =
"[{\"prop\":\"a\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"aa\"},{\"prop\":\"c\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"cc\"}]";
Assertions.assertEquals(except,
TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate));
TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator));
}
@Test
void TestGenerateCacheKey() {
String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
Property propertyI = new Property();
propertyI.setProp("i");
propertyI.setDirect(Direct.IN);
propertyI.setType(DataType.VARCHAR);
propertyI.setValue("ii");
taskExecutionContext.getPrepareParamsMap().put("i", propertyI);
String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// i will not influence the result, because task instance not use it
Assertions.assertEquals(cacheKeyBase, cacheKeyNew);
@ -157,17 +157,17 @@ class TaskCacheUtilsTest {
propertyD.setType(DataType.VARCHAR);
propertyD.setValue("dd");
taskExecutionContext.getPrepareParamsMap().put("i", propertyD);
String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// d will influence the result, because task instance use it
Assertions.assertNotEquals(cacheKeyBase, cacheKeyD);
taskInstance.setTaskDefinitionVersion(100);
String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// task definition version is changed, so cache key changed
Assertions.assertNotEquals(cacheKeyD, cacheKeyE);
taskInstance.setEnvironmentConfig("export PYTHON_LAUNCHER=/bin/python3");
String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate);
String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator);
// EnvironmentConfig is changed, so cache key changed
Assertions.assertNotEquals(cacheKeyE, cacheKeyF);
}
@ -193,7 +193,7 @@ class TaskCacheUtilsTest {
taskExecutionContext.setExecutePath("test");
taskExecutionContext.setTenantCode("aaa");
String crc = TaskCacheUtils.getValCheckSum(property, taskExecutionContext, storageOperate);
String crc = TaskCacheUtils.getValCheckSum(property, taskExecutionContext, storageOperator);
Assertions.assertEquals(crc, content);
}

6
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
View File

@ -29,7 +29,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_US
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESOURCE_UPLOAD_PATH;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.enums.StorageType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.commons.lang3.StringUtils;
@ -72,9 +72,9 @@ public class CommonUtils {
*/
public static boolean getKerberosStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
StorageType storageType = StorageType.valueOf(resUploadStartupType);
Boolean kerberosStartupState = PropertyUtils.getBoolean(HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
return resUploadType == ResUploadType.HDFS && kerberosStartupState;
return storageType == StorageType.HDFS && kerberosStartupState;
}
/**

6
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java
View File

@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.security;
import static org.apache.dolphinscheduler.common.constants.Constants.JAVA_SECURITY_KRB5_CONF;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.enums.StorageType;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@ -120,10 +120,10 @@ public class UserGroupInformationFactory {
public static boolean openKerberos() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
StorageType storageType = StorageType.valueOf(resUploadStartupType);
Boolean kerberosStartupState =
PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
return resUploadType == ResUploadType.HDFS && kerberosStartupState;
return storageType == StorageType.HDFS && kerberosStartupState;
}
}

9
dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java
View File

@ -47,4 +47,13 @@ public enum ResourceType {
public String getDescp() {
return descp;
}
public static ResourceType getResourceType(int code) {
for (ResourceType resourceType : ResourceType.values()) {
if (resourceType.getCode() == code) {
return resourceType;
}
}
return null;
}
}

368
dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java
View File

@ -19,17 +19,12 @@ package org.apache.dolphinscheduler.plugin.storage.abs;
import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING;
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
@ -37,136 +32,69 @@ import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Date;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import com.azure.core.http.rest.PagedIterable;
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.models.BlobContainerItem;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.specialized.BlockBlobClient;
@Data
@Slf4j
public class AbsStorageOperator implements Closeable, StorageOperate {
public class AbsStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator {
private BlobContainerClient blobContainerClient;
private final BlobContainerClient blobContainerClient;
private BlobServiceClient blobServiceClient;
private final BlobServiceClient blobServiceClient;
private String connectionString;
private String storageAccountName;
private String containerName;
public AbsStorageOperator() {
}
public void init() {
containerName = readContainerName();
connectionString = readConnectionString();
storageAccountName = readAccountName();
blobServiceClient = buildBlobServiceClient();
blobContainerClient = buildBlobContainerClient();
checkContainerNameExists();
}
protected BlobServiceClient buildBlobServiceClient() {
return new BlobServiceClientBuilder()
.endpoint("https://" + storageAccountName + ".blob.core.windows.net/")
.connectionString(connectionString)
public AbsStorageOperator(AbsStorageProperties absStorageProperties) {
super(absStorageProperties.getResourceUploadPath());
blobServiceClient = new BlobServiceClientBuilder()
.endpoint("https://" + absStorageProperties.getStorageAccountName() + ".blob.core.windows.net/")
.connectionString(absStorageProperties.getConnectionString())
.buildClient();
}
protected BlobContainerClient buildBlobContainerClient() {
return blobServiceClient.getBlobContainerClient(containerName);
}
protected String readConnectionString() {
return PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONNECTION_STRING);
}
protected String readContainerName() {
return PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONTAINER_NAME);
}
protected String readAccountName() {
return PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_ACCOUNT_NAME);
blobContainerClient = blobServiceClient.getBlobContainerClient(absStorageProperties.getContainerName());
checkContainerNameExists(absStorageProperties.getContainerName());
}
@Override
public void createTenantDirIfNotExists(String tenantCode) throws Exception {
mkdir(tenantCode, getAbsResDir(tenantCode));
mkdir(tenantCode, getAbsUdfDir(tenantCode));
}
public String getAbsResDir(String tenantCode) {
return String.format("%s/" + RESOURCE_TYPE_FILE, getAbsTenantDir(tenantCode));
}
public String getAbsUdfDir(String tenantCode) {
return String.format("%s/" + RESOURCE_TYPE_UDF, getAbsTenantDir(tenantCode));
}
public String getAbsTenantDir(String tenantCode) {
return String.format(FORMAT_S_S, getGcsDataBasePath(), tenantCode);
}
public String getGcsDataBasePath() {
if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) {
return EMPTY_STRING;
} else {
return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING);
}
public String getStorageBaseDirectory() {
// All directory should end with File.separator
if (getStorageBaseDirectory().startsWith("/")) {
log.warn("{} -> {} should not start with / in abs", Constants.RESOURCE_UPLOAD_PATH,
getStorageBaseDirectory());
return getStorageBaseDirectory().substring(1);
}
return getStorageBaseDirectory();
}
@SneakyThrows
@Override
public String getResDir(String tenantCode) {
return getAbsResDir(tenantCode) + FOLDER_SEPARATOR;
public void createStorageDir(String directory) {
String objectName = directory + FOLDER_SEPARATOR;
if (isObjectExists(objectName)) {
throw new FileAlreadyExistsException("directory: " + objectName + " already exists");
}
BlobClient blobClient = blobContainerClient.getBlobClient(objectName);
blobClient.upload(new ByteArrayInputStream(EMPTY_STRING.getBytes()), 0);
}
@SneakyThrows
@Override
public String getUdfDir(String tenantCode) {
return getAbsUdfDir(tenantCode) + FOLDER_SEPARATOR;
}
@Override
public String getResourceFullName(String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING);
}
return String.format(FORMAT_S_S, getAbsResDir(tenantCode), fileName);
}
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
fileName = fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING);
}
return getDir(resourceType, tenantCode) + fileName;
}
@Override
public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException {
public void download(String srcFilePath, String dstFilePath, boolean overwrite) {
File dstFile = new File(dstFilePath);
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
@ -179,7 +107,7 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
}
@Override
public boolean exists(String fullName) throws IOException {
public boolean exists(String fullName) {
return isObjectExists(fullName);
}
@ -187,36 +115,14 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
return blobContainerClient.getBlobClient(objectName).exists();
}
@SneakyThrows
@Override
public boolean delete(String filePath, boolean recursive) throws IOException {
try {
if (isObjectExists(filePath)) {
blobContainerClient.getBlobClient(filePath).delete();
}
return true;
} catch (Exception e) {
log.error("delete the object error,the resource path is {}", filePath);
return false;
}
public void delete(String filePath, boolean recursive) {
blobContainerClient.getBlobClient(filePath).deleteIfExists();
}
@Override
public boolean delete(String fullName, List<String> childrenPathList, boolean recursive) throws IOException {
// append the resource fullName to the list for deletion.
childrenPathList.add(fullName);
boolean result = true;
for (String filePath : childrenPathList) {
if (!delete(filePath, recursive)) {
result = false;
}
}
return result;
}
@Override
public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException {
public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) {
BlobClient srcBlobClient = blobContainerClient.getBlobClient(srcPath);
BlockBlobClient dstBlobClient = blobContainerClient.getBlobClient(dstPath).getBlockBlobClient();
@ -225,13 +131,11 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
if (deleteSource) {
srcBlobClient.delete();
}
return true;
}
@SneakyThrows
@Override
public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource,
boolean overwrite) throws IOException {
try {
public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) {
BlobClient blobClient = blobContainerClient.getBlobClient(dstPath);
blobClient.uploadFromFile(srcFile, overwrite);
@ -239,15 +143,11 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
if (deleteSource) {
Files.delete(srcPath);
}
return true;
} catch (Exception e) {
log.error("upload failed,the container is {},the filePath is {}", containerName, dstPath);
return false;
}
}
@SneakyThrows
@Override
public List<String> vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException {
public List<String> fetchFileContent(String filePath, int skipLineNums, int limit) {
if (StringUtils.isBlank(filePath)) {
log.error("file path:{} is blank", filePath);
return Collections.emptyList();
@ -263,199 +163,27 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
}
}
@Override
public void deleteTenant(String tenantCode) throws Exception {
deleteTenantCode(tenantCode);
}
protected void deleteTenantCode(String tenantCode) {
deleteDirectory(getResDir(tenantCode));
deleteDirectory(getUdfDir(tenantCode));
}
@Override
public String getDir(ResourceType resourceType, String tenantCode) {
switch (resourceType) {
case UDF:
return getUdfDir(tenantCode);
case FILE:
return getResDir(tenantCode);
case ALL:
return getGcsDataBasePath();
default:
return EMPTY_STRING;
}
}
protected void deleteDirectory(String directoryName) {
if (isObjectExists(directoryName)) {
blobContainerClient.getBlobClient(directoryName).delete();
}
}
@Override
public boolean mkdir(String tenantCode, String path) throws IOException {
String objectName = path + FOLDER_SEPARATOR;
if (!isObjectExists(objectName)) {
BlobClient blobClient = blobContainerClient.getBlobClient(objectName);
blobClient.upload(new ByteArrayInputStream(EMPTY_STRING.getBytes()), 0);
}
return true;
}
@Override
public void close() throws IOException {
}
@Override
public ResUploadType returnStorageType() {
return ResUploadType.ABS;
public List<StorageEntity> listStorageEntity(String resourceAbsolutePath) {
return null;
}
@Override
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type) {
List<StorageEntity> storageEntityList = new ArrayList<>();
LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
StorageEntity initialEntity = null;
try {
initialEntity = getFileStatus(path, defaultPath, tenantCode, type);
} catch (Exception e) {
log.error("error while listing files status recursively, path: {}", path, e);
return storageEntityList;
}
foldersToFetch.add(initialEntity);
while (!foldersToFetch.isEmpty()) {
String pathToExplore = foldersToFetch.pop().getFullName();
try {
List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
for (StorageEntity temp : tempList) {
if (temp.isDirectory()) {
foldersToFetch.add(temp);
}
}
storageEntityList.addAll(tempList);
} catch (Exception e) {
log.error("error while listing files stat:wus recursively, path: {}", pathToExplore, e);
}
}
return storageEntityList;
public List<StorageEntity> listFileStorageEntityRecursively(String resourceAbsolutePath) {
return null;
}
@Override
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
List<StorageEntity> storageEntityList = new ArrayList<>();
PagedIterable<BlobItem> blobItems;
blobItems = blobContainerClient.listBlobsByHierarchy(path);
if (blobItems == null) {
return storageEntityList;
public StorageEntity getStorageEntity(String resourceAbsolutePath) {
return null;
}
for (BlobItem blobItem : blobItems) {
if (path.equals(blobItem.getName())) {
continue;
}
if (blobItem.isPrefix()) {
String suffix = StringUtils.difference(path, blobItem.getName());
String fileName = StringUtils.difference(defaultPath, blobItem.getName());
StorageEntity entity = new StorageEntity();
entity.setAlias(suffix);
entity.setFileName(fileName);
entity.setFullName(blobItem.getName());
entity.setDirectory(true);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
entity.setCreateTime(null);
entity.setUpdateTime(null);
entity.setPfullName(path);
storageEntityList.add(entity);
} else {
String[] aliasArr = blobItem.getName().split("/");
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, blobItem.getName());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(blobItem.getName());
entity.setDirectory(false);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(blobItem.getProperties().getContentLength());
entity.setCreateTime(Date.from(blobItem.getProperties().getCreationTime().toInstant()));
entity.setUpdateTime(Date.from(blobItem.getProperties().getLastModified().toInstant()));
entity.setPfullName(path);
storageEntityList.add(entity);
}
}
return storageEntityList;
}
@Override
public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
if (path.endsWith(FOLDER_SEPARATOR)) {
// the path is a directory that may or may not exist
String alias = findDirAlias(path);
String fileName = StringUtils.difference(defaultPath, path);
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(path);
entity.setDirectory(true);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
return entity;
} else {
if (isObjectExists(path)) {
BlobClient blobClient = blobContainerClient.getBlobClient(path);
String[] aliasArr = blobClient.getBlobName().split(FOLDER_SEPARATOR);
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, blobClient.getBlobName());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(blobClient.getBlobName());
entity.setDirectory(false);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(blobClient.getProperties().getBlobSize());
entity.setCreateTime(Date.from(blobClient.getProperties().getCreationTime().toInstant()));
entity.setUpdateTime(Date.from(blobClient.getProperties().getLastModified().toInstant()));
return entity;
} else {
throw new FileNotFoundException("Object is not found in ABS container: " + containerName);
}
}
}
private String findDirAlias(String dirPath) {
if (!dirPath.endsWith(FOLDER_SEPARATOR)) {
return dirPath;
}
Path path = Paths.get(dirPath);
return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR;
}
public void checkContainerNameExists() {
public void checkContainerNameExists(String containerName) {
if (StringUtils.isBlank(containerName)) {
throw new IllegalArgumentException(containerName + " is blank");
}

26
dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java
View File

@ -17,20 +17,30 @@
package org.apache.dolphinscheduler.plugin.storage.abs;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory;
import org.apache.dolphinscheduler.plugin.storage.api.StorageType;
import com.google.auto.service.AutoService;
@AutoService(StorageOperateFactory.class)
public class AbsStorageOperatorFactory implements StorageOperateFactory {
@AutoService(StorageOperatorFactory.class)
public class AbsStorageOperatorFactory implements StorageOperatorFactory {
@Override
public StorageOperate createStorageOperate() {
AbsStorageOperator absStorageOperator = new AbsStorageOperator();
absStorageOperator.init();
return absStorageOperator;
public StorageOperator createStorageOperate() {
final AbsStorageProperties absStorageProperties = getAbsStorageProperties();
return new AbsStorageOperator(absStorageProperties);
}
private AbsStorageProperties getAbsStorageProperties() {
return AbsStorageProperties.builder()
.containerName(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONTAINER_NAME))
.connectionString(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONNECTION_STRING))
.storageAccountName(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_ACCOUNT_NAME))
.resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"))
.build();
}
@Override

36
dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageProperties.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.abs;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class AbsStorageProperties {
private String containerName;
private String connectionString;
private String storageAccountName;
private String resourceUploadPath;
}

274
dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/test/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorTest.java
View File

@ -1,274 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.abs;
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.specialized.BlockBlobClient;
@ExtendWith(MockitoExtension.class)
public class AbsStorageOperatorTest {
private static final String CONNECTION_STRING_MOCK = "CONNECTION_STRING_MOCK";
private static final String ACCOUNT_NAME_MOCK = "ACCOUNT_NAME_MOCK";
private static final String CONTAINER_NAME_MOCK = "CONTAINER_NAME_MOCK";
private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK";
private static final String DIR_MOCK = "DIR_MOCK";
private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK";
private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK";
private static final String FULL_NAME = "/tmp/dir1/";
private static final String DEFAULT_PATH = "/tmp/";
@Mock
private BlobContainerClient blobContainerClient;
@Mock
private BlobServiceClient blobServiceClient;
@Mock
private BlockBlobClient blockBlobClient;
@Mock
private BlobClient blobClient;
private AbsStorageOperator absStorageOperator;
@BeforeEach
public void setUp() throws Exception {
absStorageOperator = Mockito.spy(AbsStorageOperator.class);
Mockito.doReturn(CONNECTION_STRING_MOCK).when(absStorageOperator).readConnectionString();
Mockito.doReturn(CONTAINER_NAME_MOCK).when(absStorageOperator).readContainerName();
Mockito.doReturn(ACCOUNT_NAME_MOCK).when(absStorageOperator).readAccountName();
Mockito.doReturn(blobContainerClient).when(absStorageOperator).buildBlobContainerClient();
Mockito.doReturn(blobServiceClient).when(absStorageOperator).buildBlobServiceClient();
Mockito.doNothing().when(absStorageOperator).checkContainerNameExists();
absStorageOperator.init();
}
@Test
public void testInit() throws Exception {
verify(absStorageOperator, times(1)).buildBlobServiceClient();
verify(absStorageOperator, times(1)).buildBlobContainerClient();
Assertions.assertEquals(CONNECTION_STRING_MOCK, absStorageOperator.getConnectionString());
Assertions.assertEquals(CONTAINER_NAME_MOCK, absStorageOperator.getContainerName());
Assertions.assertEquals(ACCOUNT_NAME_MOCK, absStorageOperator.getStorageAccountName());
}
@Test
public void createTenantResAndUdfDir() throws Exception {
doReturn(DIR_MOCK).when(absStorageOperator).getAbsResDir(TENANT_CODE_MOCK);
doReturn(DIR_MOCK).when(absStorageOperator).getAbsUdfDir(TENANT_CODE_MOCK);
doReturn(true).when(absStorageOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK);
absStorageOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK);
verify(absStorageOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK);
}
@Test
public void getResDir() {
final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK);
final String dir = absStorageOperator.getResDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedResourceDir, dir);
}
@Test
public void getUdfDir() {
final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK);
final String dir = absStorageOperator.getUdfDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedUdfDir, dir);
}
@Test
public void mkdirWhenDirExists() {
boolean isSuccess = false;
try {
final String key = DIR_MOCK + FOLDER_SEPARATOR;
Mockito.doReturn(true).when(absStorageOperator).isObjectExists(key);
isSuccess = absStorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK);
} catch (IOException e) {
Assertions.fail("test failed due to unexpected IO exception");
}
Assertions.assertTrue(isSuccess);
}
@Test
public void getResourceFullName() {
final String expectedResourceFileName =
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
final String resourceFileName = absStorageOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
Assertions.assertEquals(expectedResourceFileName, resourceFileName);
}
@Test
public void getFileName() {
final String expectedFileName =
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
final String fileName = absStorageOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK);
Assertions.assertEquals(expectedFileName, fileName);
}
@Test
public void exists() {
boolean doesExist = false;
doReturn(true).when(absStorageOperator).isObjectExists(FILE_NAME_MOCK);
try {
doesExist = absStorageOperator.exists(FILE_NAME_MOCK);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
Assertions.assertTrue(doesExist);
}
@Test
public void delete() {
boolean isDeleted = false;
doReturn(true).when(absStorageOperator).isObjectExists(FILE_NAME_MOCK);
Mockito.doReturn(blobClient).when(blobContainerClient).getBlobClient(Mockito.anyString());
try {
isDeleted = absStorageOperator.delete(FILE_NAME_MOCK, true);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
Assertions.assertTrue(isDeleted);
verify(blobClient, times(1)).delete();
}
@Test
public void copy() {
boolean isSuccess = false;
Mockito.doReturn(blobClient).when(blobContainerClient).getBlobClient(Mockito.anyString());
Mockito.doReturn(blockBlobClient).when(blobClient).getBlockBlobClient();
try {
isSuccess = absStorageOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
Assertions.assertTrue(isSuccess);
}
@Test
public void deleteTenant() {
doNothing().when(absStorageOperator).deleteTenantCode(anyString());
try {
absStorageOperator.deleteTenant(TENANT_CODE_MOCK);
} catch (Exception e) {
Assertions.fail("unexpected exception caught in unit test");
}
verify(absStorageOperator, times(1)).deleteTenantCode(anyString());
}
@Test
public void getGcsResDir() {
final String expectedGcsResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK);
final String gcsResDir = absStorageOperator.getAbsResDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedGcsResDir, gcsResDir);
}
@Test
public void getGcsUdfDir() {
final String expectedGcsUdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK);
final String gcsUdfDir = absStorageOperator.getAbsUdfDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedGcsUdfDir, gcsUdfDir);
}
@Test
public void getGcsTenantDir() {
final String expectedGcsTenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK);
doReturn(DIR_MOCK).when(absStorageOperator).getGcsDataBasePath();
final String gcsTenantDir = absStorageOperator.getAbsTenantDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedGcsTenantDir, gcsTenantDir);
}
@Test
public void deleteDir() {
Mockito.doReturn(blobClient).when(blobContainerClient).getBlobClient(Mockito.anyString());
doReturn(true).when(absStorageOperator).isObjectExists(Mockito.any());
absStorageOperator.deleteDirectory(DIR_MOCK);
verify(blobClient, times(1)).delete();
}
@Test
public void testGetFileStatus() throws Exception {
StorageEntity entity =
absStorageOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
Assertions.assertEquals(FULL_NAME, entity.getFullName());
Assertions.assertEquals("dir1/", entity.getFileName());
}
@Test
public void testListFilesStatus() throws Exception {
Mockito.doReturn(null).when(blobContainerClient).listBlobsByHierarchy(Mockito.any());
List<StorageEntity> result =
absStorageOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
verify(blobContainerClient, times(1)).listBlobsByHierarchy(Mockito.any());
}
@Test
public void testListFilesStatusRecursively() throws Exception {
StorageEntity entity = new StorageEntity();
entity.setFullName(FULL_NAME);
doReturn(entity).when(absStorageOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK,
ResourceType.FILE);
doReturn(Collections.EMPTY_LIST).when(absStorageOperator).listFilesStatus(anyString(), anyString(), anyString(),
Mockito.any(ResourceType.class));
List<StorageEntity> result =
absStorageOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK,
ResourceType.FILE);
Assertions.assertEquals(0, result.size());
}
}

110
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.api;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import com.google.common.base.Preconditions;
import com.google.common.io.Files;
public abstract class AbstractStorageOperator implements StorageOperator {
protected final String resourceBaseAbsolutePath;
public AbstractStorageOperator(String resourceBaseAbsolutePath) {
Preconditions.checkNotNull(resourceBaseAbsolutePath, "Resource upload path should not be null");
this.resourceBaseAbsolutePath = resourceBaseAbsolutePath;
}
@Override
public ResourceMetadata getResourceMetaData(String resourceAbsolutePath) {
String storageBaseDirectory = getStorageBaseDirectory();
String resourceSegment = StringUtils.substringAfter(resourceAbsolutePath, storageBaseDirectory);
String[] segments = StringUtils.split(resourceSegment, File.separator, 3);
if (segments.length == 0) {
throw new IllegalArgumentException("Invalid resource path: " + resourceAbsolutePath);
}
return ResourceMetadata.builder()
.resourceAbsolutePath(resourceAbsolutePath)
.resourceBaseDirectory(storageBaseDirectory)
.isDirectory(Files.getFileExtension(resourceAbsolutePath).isEmpty())
.tenant(segments[0])
.resourceType(segments[1].equals(FILE_FOLDER_NAME) ? ResourceType.FILE : ResourceType.UDF)
.resourceRelativePath(segments.length == 2 ? "/" : segments[2])
.resourceParentAbsolutePath(StringUtils.substringBeforeLast(resourceAbsolutePath, File.separator))
.build();
}
@Override
public String getStorageBaseDirectory() {
// All directory should end with File.separator
return PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
}
@Override
public String getStorageBaseDirectory(String tenantCode) {
if (StringUtils.isEmpty(tenantCode)) {
throw new IllegalArgumentException("Tenant code should not be empty");
}
// All directory should end with File.separator
return FileUtils.concatFilePath(getStorageBaseDirectory(), tenantCode);
}
@Override
public String getStorageBaseDirectory(String tenantCode, ResourceType resourceType) {
String tenantBaseDirectory = getStorageBaseDirectory(tenantCode);
if (resourceType == null) {
throw new IllegalArgumentException("Resource type should not be null");
}
String resourceBaseDirectory;
switch (resourceType) {
case FILE:
resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, FILE_FOLDER_NAME);
break;
case UDF:
resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, UDF_FOLDER_NAME);
break;
case ALL:
resourceBaseDirectory = tenantBaseDirectory;
break;
default:
throw new IllegalArgumentException("Resource type: " + resourceType + " not supported");
}
// All directory should end with File.separator
return resourceBaseDirectory;
}
@Override
public String getStorageFileAbsolutePath(String tenantCode, String fileName) {
return FileUtils.concatFilePath(getStorageBaseDirectory(tenantCode, ResourceType.FILE), fileName);
}
protected void exceptionIfPathEmpty(String resourceAbsolutePath) {
if (StringUtils.isEmpty(resourceAbsolutePath)) {
throw new IllegalArgumentException("Resource path should not be empty");
}
}
}

42
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.api;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ResourceMetadata {
private String resourceAbsolutePath;
private String resourceBaseDirectory;
private String tenant;
private ResourceType resourceType;
private String resourceRelativePath;
private String resourceParentAbsolutePath;
private boolean isDirectory;
}

10
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java
View File

@ -32,13 +32,13 @@ import org.springframework.context.annotation.Configuration;
public class StorageConfiguration {
@Bean
public StorageOperate storageOperate() {
public StorageOperator storageOperate() {
Optional<StorageType> storageTypeOptional =
StorageType.getStorageType(PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE));
Optional<StorageOperate> storageOperate = storageTypeOptional.map(storageType -> {
ServiceLoader<StorageOperateFactory> storageOperateFactories =
ServiceLoader.load(StorageOperateFactory.class);
for (StorageOperateFactory storageOperateFactory : storageOperateFactories) {
Optional<StorageOperator> storageOperate = storageTypeOptional.map(storageType -> {
ServiceLoader<StorageOperatorFactory> storageOperateFactories =
ServiceLoader.load(StorageOperatorFactory.class);
for (StorageOperatorFactory storageOperateFactory : storageOperateFactories) {
if (storageOperateFactory.getStorageOperate() == storageType) {
return storageOperateFactory.createStorageOperate();
}

27
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java
View File

@ -24,7 +24,10 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.util.Date;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
// StorageEneity is an entity representing a resource in the third-part storage service.
// It is only stored in t_ds_relation_resources_task if the resource is used by a task.
@ -32,32 +35,16 @@ import lombok.Data;
// in table t_ds_relation_resources_task.
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class StorageEntity {
/**
* exist only if it is stored in t_ds_relation_resources_task.
*
*/
private int id;
/**
* fullname is in a format of basepath + tenantCode + res/udf + filename
*/
private String fullName;
/**
* filename is in a format of possible parent folders + alias
*/
private String fileName;
/**
* the name of the file
*/
private String alias;
/**
* parent folder time
*/
private String pfullName;
private boolean isDirectory;
private int userId;
private String userName;
private ResourceType type;
private long size;
private Date createTime;

203
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java
View File

@ -1,203 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.api;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.io.IOException;
import java.util.List;
public interface StorageOperate {
String RESOURCE_UPLOAD_PATH = PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
/**
* if the resource of tenant 's exist, the resource of folder will be created
* @param tenantCode
* @throws Exception
*/
void createTenantDirIfNotExists(String tenantCode) throws Exception;
/**
* get the resource directory of tenant
* @param tenantCode
* @return
*/
String getResDir(String tenantCode);
/**
* return the udf directory of tenant
* @param tenantCode
* @return
*/
String getUdfDir(String tenantCode);
/**
* create the directory that the path of tenant wanted to create
* @param tenantCode
* @param path
* @return
* @throws IOException
*/
boolean mkdir(String tenantCode, String path) throws IOException;
/**
* get the path of the resource file (fullName)
* @param tenantCode
* @param fileName
* @return
*/
String getResourceFullName(String tenantCode, String fileName);
/**
* get the path of the resource file excluding the base path (fileName)
*/
default String getResourceFileName(String tenantCode, String fullName) {
String resDir = getResDir(tenantCode);
String filenameReplaceResDir = fullName.replaceFirst(resDir, "");
if (!filenameReplaceResDir.equals(fullName)) {
return filenameReplaceResDir;
}
// Replace resource dir not effective in case of run workflow with different tenant from resource file's.
// this is backup solution to get related path, by split with RESOURCE_TYPE_FILE
return filenameReplaceResDir.contains(RESOURCE_TYPE_FILE)
? filenameReplaceResDir.split(String.format("%s/", RESOURCE_TYPE_FILE))[1]
: filenameReplaceResDir;
}
/**
* get the path of the file
* @param resourceType
* @param tenantCode
* @param fileName
* @return
*/
String getFileName(ResourceType resourceType, String tenantCode, String fileName);
/**
* predicate if the resource of tenant exists
* @param fullName
* @return
* @throws IOException
*/
boolean exists(String fullName) throws IOException;
/**
* delete the resource of filePath
* todo if the filePath is the type of directory ,the files in the filePath need to be deleted at all
* @param filePath
* @param recursive
* @return
* @throws IOException
*/
boolean delete(String filePath, boolean recursive) throws IOException;
boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException;
/**
* copy the file from srcPath to dstPath
* @param srcPath
* @param dstPath
* @param deleteSource if need to delete the file of srcPath
* @param overwrite
* @return
* @throws IOException
*/
boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException;
/**
* get the root path of the tenant with resourceType
* @param resourceType
* @param tenantCode
* @return
*/
String getDir(ResourceType resourceType, String tenantCode);
/**
* upload the local srcFile to dstPath
* @param tenantCode
* @param srcFile
* @param dstPath
* @param deleteSource
* @param overwrite
* @return
* @throws IOException
*/
boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource,
boolean overwrite) throws IOException;
/**
* download the srcPath to local
*
* @param srcFilePath the full path of the srcPath
* @param dstFile
* @param overwrite
* @throws IOException
*/
void download(String srcFilePath, String dstFile, boolean overwrite) throws IOException;
/**
* vim the context of filePath
* @param tenantCode
* @param filePath
* @param skipLineNums
* @param limit
* @return
* @throws IOException
*/
List<String> vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException;
/**
* delete the files and directory of the tenant
*
* @param tenantCode
* @throws Exception
*/
void deleteTenant(String tenantCode) throws Exception;
/**
* return the storageType
*
* @return
*/
ResUploadType returnStorageType();
/**
* return files and folders in the current directory and subdirectories
* */
List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type);
/**
* return files and folders in the current directory
* */
List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception;
/**
* return a file status
* */
StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception;
}

157
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java Normal file
View File

@ -0,0 +1,157 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.api;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.nio.file.FileAlreadyExistsException;
import java.util.List;
public interface StorageOperator {
String FILE_FOLDER_NAME = "resources";
String UDF_FOLDER_NAME = "udfs";
ResourceMetadata getResourceMetaData(String resourceAbsolutePath);
/**
* Get the absolute path of base directory.
*
* @return the base directory. e.g. file:///tmp/dolphinscheduler/, /tmp/dolphinscheduler/
*/
String getStorageBaseDirectory();
/**
* Get the absolute path of directory which will be used by the given tenant. the tenant directory is under the base directory.
*
* @param tenantCode the tenant code, cannot be empty
* @return the tenant directory. e.g. file:///tmp/dolphinscheduler/default/
*/
String getStorageBaseDirectory(String tenantCode);
/**
* Get the absolute path of directory which will be used by the given tenant and resource type. the resource directory is under the tenant directory.
* <p> If the resource type is FILE, will be 'file:///tmp/dolphinscheduler/default/resources/'.
* <p> If the resource type is UDF, will be 'is file:///tmp/dolphinscheduler/default/udfs/'.
* <p> If the resource type is ALL, will be 'is file:///tmp/dolphinscheduler/default/'.
*
* @param tenantCode the tenant code, cannot be empty
* @param resourceType the resource type, cannot be null
* @return the resource directory. e.g. file:///tmp/dolphinscheduler/default/resources/
*/
String getStorageBaseDirectory(String tenantCode, ResourceType resourceType);
/**
* Get the absolute path of the file in the storage. the file will under the file resource directory.
*
* @param tenantCode the tenant code, cannot be empty
* @param fileName the file name, cannot be empty
* @return the file absolute path. e.g. file:///tmp/dolphinscheduler/default/resources/test.sh
*/
String getStorageFileAbsolutePath(String tenantCode, String fileName);
/**
* Create a directory if the directory is already exists will throw exception(Dependent on the storage implementation).
* <p> If the directory is not exists, will create the directory.
* <p> If the parent directory is not exists, will create the parent directory.
* <p> If the directory is already exists, will throw {@link FileAlreadyExistsException}.
*
* @param directoryAbsolutePath the directory absolute path
*/
void createStorageDir(String directoryAbsolutePath);
/**
* Check if the resource exists.
*
* @param resourceAbsolutePath the resource absolute path
* @return true if the resource exists, otherwise false
*/
boolean exists(String resourceAbsolutePath);
/**
* Delete the resource, if the resourceAbsolutePath is not exists, will do nothing.
*
* @param resourceAbsolutePath the resource absolute path
* @param recursive whether to delete all the sub file/directory under the given resource
*/
void delete(String resourceAbsolutePath, boolean recursive);
/**
* Copy the resource from the source path to the destination path.
*
* @param srcAbsolutePath the source path
* @param dstAbsolutePath the destination path
* @param deleteSource whether to delete the source path after copying
* @param overwrite whether to overwrite the destination path if it exists
*/
void copy(String srcAbsolutePath, String dstAbsolutePath, boolean deleteSource, boolean overwrite);
/**
* Move the resource from the source path to the destination path.
*
* @param srcLocalFileAbsolutePath the source local file
* @param dstAbsolutePath the destination path
* @param deleteSource whether to delete the source path after moving
* @param overwrite whether to overwrite the destination path if it exists
*/
void upload(String srcLocalFileAbsolutePath, String dstAbsolutePath, boolean deleteSource, boolean overwrite);
/**
* Download the resource from the source path to the destination path.
*
* @param srcFileAbsolutePath the source path
* @param dstAbsoluteFile the destination file
* @param overwrite whether to overwrite the destination file if it exists
*/
void download(String srcFileAbsolutePath, String dstAbsoluteFile, boolean overwrite);
/**
* Fetch the content of the file.
*
* @param fileAbsolutePath the file path
* @param skipLineNums the number of lines to skip
* @param limit the number of lines to read
* @return the content of the file
*/
List<String> fetchFileContent(String fileAbsolutePath, int skipLineNums, int limit);
/**
* Return the {@link StorageEntity} under the given path.
* <p>If the path is a file, return the file status.
* <p>If the path is a directory, return the file/directory under the directory.
* <p>If the path is not exist, will return empty.
*
* @param resourceAbsolutePath the resource absolute path, cannot be empty
*/
List<StorageEntity> listStorageEntity(String resourceAbsolutePath);
/**
* Return the {@link StorageEntity} which is file under the given path
*
* @param resourceAbsolutePath the resource absolute path, cannot be empty
*/
List<StorageEntity> listFileStorageEntityRecursively(String resourceAbsolutePath);
/**
* Return the {@link StorageEntity} under the current directory
*
* @param resourceAbsolutePath the resource absolute path, cannot be empty
*/
StorageEntity getStorageEntity(String resourceAbsolutePath);
}

4
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperateFactory.java → dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperatorFactory.java
View File

@ -17,9 +17,9 @@
package org.apache.dolphinscheduler.plugin.storage.api;
public interface StorageOperateFactory {
public interface StorageOperatorFactory {
StorageOperate createStorageOperate();
StorageOperator createStorageOperate();
StorageType getStorageOperate();
}

418
dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
View File

@ -18,18 +18,13 @@
package org.apache.dolphinscheduler.plugin.storage.gcs;
import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING;
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
@ -37,22 +32,24 @@ import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Date;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import com.google.api.gax.paging.Page;
@ -64,82 +61,53 @@ import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
@Data
@Slf4j
public class GcsStorageOperator implements Closeable, StorageOperate {
public class GcsStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator {
private Storage gcsStorage;
private final Storage gcsStorage;
private String bucketName;
private final String bucketName;
private String credential;
public GcsStorageOperator() {
}
public void init() {
try {
credential = readCredentials();
bucketName = readBucketName();
gcsStorage = buildGcsStorage(credential);
checkBucketNameExists(bucketName);
} catch (IOException e) {
log.error("GCS Storage operator init failed", e);
}
}
protected Storage buildGcsStorage(String credential) throws IOException {
return StorageOptions.newBuilder()
@SneakyThrows
public GcsStorageOperator(GcsStorageProperties gcsStorageProperties) {
super(gcsStorageProperties.getResourceUploadPath());
bucketName = gcsStorageProperties.getBucketName();
gcsStorage = StorageOptions.newBuilder()
.setCredentials(ServiceAccountCredentials.fromStream(
Files.newInputStream(Paths.get(credential))))
Files.newInputStream(Paths.get(gcsStorageProperties.getCredential()))))
.build()
.getService();
}
protected String readCredentials() {
return PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_CREDENTIAL);
}
protected String readBucketName() {
return PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME);
checkBucketNameExists(bucketName);
}
@Override
public void createTenantDirIfNotExists(String tenantCode) throws Exception {
mkdir(tenantCode, getGcsResDir(tenantCode));
mkdir(tenantCode, getGcsUdfDir(tenantCode));
public String getStorageBaseDirectory() {
// All directory should end with File.separator
if (resourceBaseAbsolutePath.startsWith("/")) {
log.warn("{} -> {} should not start with / in Gcs", Constants.RESOURCE_UPLOAD_PATH,
resourceBaseAbsolutePath);
return resourceBaseAbsolutePath.substring(1);
}
return getStorageBaseDirectory();
}
@SneakyThrows
@Override
public String getResDir(String tenantCode) {
return getGcsResDir(tenantCode) + FOLDER_SEPARATOR;
public void createStorageDir(String directoryAbsolutePath) {
directoryAbsolutePath = transformAbsolutePathToGcsKey(directoryAbsolutePath);
if (exists(directoryAbsolutePath)) {
throw new FileAlreadyExistsException("directory: " + directoryAbsolutePath + " already exists");
}
BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(bucketName, directoryAbsolutePath)).build();
gcsStorage.create(blobInfo, EMPTY_STRING.getBytes(StandardCharsets.UTF_8));
}
@SneakyThrows
@Override
public String getUdfDir(String tenantCode) {
return getGcsUdfDir(tenantCode) + FOLDER_SEPARATOR;
}
public void download(String srcFilePath, String dstFilePath, boolean overwrite) {
srcFilePath = transformAbsolutePathToGcsKey(srcFilePath);
@Override
public String getResourceFullName(String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING);
}
return String.format(FORMAT_S_S, getGcsResDir(tenantCode), fileName);
}
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
fileName = fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING);
}
return getDir(resourceType, tenantCode) + fileName;
}
@Override
public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException {
File dstFile = new File(dstFilePath);
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
@ -152,40 +120,26 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
}
@Override
public boolean exists(String fullName) throws IOException {
return isObjectExists(fullName);
public boolean exists(String fullName) {
fullName = transformAbsolutePathToGcsKey(fullName);
Blob blob = gcsStorage.get(BlobId.of(bucketName, fullName));
return blob != null && blob.exists();
}
@SneakyThrows
@Override
public boolean delete(String filePath, boolean recursive) throws IOException {
try {
if (isObjectExists(filePath)) {
public void delete(String filePath, boolean recursive) {
filePath = transformAbsolutePathToGcsKey(filePath);
if (exists(filePath)) {
gcsStorage.delete(BlobId.of(bucketName, filePath));
}
return true;
} catch (Exception e) {
log.error("delete the object error,the resource path is {}", filePath);
return false;
}
}
@Override
public boolean delete(String fullName, List<String> childrenPathList, boolean recursive) throws IOException {
// append the resource fullName to the list for deletion.
childrenPathList.add(fullName);
public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) {
srcPath = transformGcsKeyToAbsolutePath(srcPath);
dstPath = transformGcsKeyToAbsolutePath(dstPath);
boolean result = true;
for (String filePath : childrenPathList) {
if (!delete(filePath, recursive)) {
result = false;
}
}
return result;
}
@Override
public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException {
BlobId source = BlobId.of(bucketName, srcPath);
BlobId target = BlobId.of(bucketName, dstPath);
@ -198,13 +152,15 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
if (deleteSource) {
gcsStorage.delete(source);
}
return true;
}
@SneakyThrows
@Override
public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource,
boolean overwrite) throws IOException {
try {
public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) {
dstPath = transformAbsolutePathToGcsKey(dstPath);
if (exists(dstPath) && !overwrite) {
throw new FileAlreadyExistsException("file: " + dstPath + " already exists");
}
BlobInfo blobInfo = BlobInfo.newBuilder(
BlobId.of(bucketName, dstPath)).build();
@ -214,15 +170,12 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
if (deleteSource) {
Files.delete(srcPath);
}
return true;
} catch (Exception e) {
log.error("upload failed,the bucketName is {},the filePath is {}", bucketName, dstPath);
return false;
}
}
@SneakyThrows
@Override
public List<String> vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException {
public List<String> fetchFileContent(String filePath, int skipLineNums, int limit) {
filePath = transformAbsolutePathToGcsKey(filePath);
if (StringUtils.isBlank(filePath)) {
log.error("file path:{} is blank", filePath);
return Collections.emptyList();
@ -237,232 +190,58 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
}
}
@Override
public void deleteTenant(String tenantCode) throws Exception {
deleteTenantCode(tenantCode);
}
protected void deleteTenantCode(String tenantCode) {
deleteDirectory(getResDir(tenantCode));
deleteDirectory(getUdfDir(tenantCode));
}
@Override
public String getDir(ResourceType resourceType, String tenantCode) {
switch (resourceType) {
case UDF:
return getUdfDir(tenantCode);
case FILE:
return getResDir(tenantCode);
case ALL:
return getGcsDataBasePath();
default:
return EMPTY_STRING;
}
}
protected void deleteDirectory(String directoryName) {
if (isObjectExists(directoryName)) {
gcsStorage.delete(BlobId.of(bucketName, directoryName));
}
}
public String getGcsResDir(String tenantCode) {
return String.format("%s/" + RESOURCE_TYPE_FILE, getGcsTenantDir(tenantCode));
}
public String getGcsUdfDir(String tenantCode) {
return String.format("%s/" + RESOURCE_TYPE_UDF, getGcsTenantDir(tenantCode));
}
public String getGcsTenantDir(String tenantCode) {
return String.format(FORMAT_S_S, getGcsDataBasePath(), tenantCode);
}
public String getGcsDataBasePath() {
if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) {
return EMPTY_STRING;
} else {
return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING);
}
}
@Override
public boolean mkdir(String tenantCode, String path) throws IOException {
String objectName = path + FOLDER_SEPARATOR;
if (!isObjectExists(objectName)) {
BlobInfo blobInfo = BlobInfo.newBuilder(
BlobId.of(bucketName, objectName)).build();
gcsStorage.create(blobInfo, EMPTY_STRING.getBytes(StandardCharsets.UTF_8));
}
return true;
}
@SneakyThrows
@Override
public void close() throws IOException {
try {
if (gcsStorage != null) {
gcsStorage.close();
}
} catch (Exception e) {
throw new IOException(e);
}
}
@Override
public ResUploadType returnStorageType() {
return ResUploadType.GCS;
public List<StorageEntity> listStorageEntity(String resourceAbsolutePath) {
resourceAbsolutePath = transformAbsolutePathToGcsKey(resourceAbsolutePath);
Page<Blob> blobs = gcsStorage.list(bucketName, Storage.BlobListOption.prefix(resourceAbsolutePath));
List<StorageEntity> storageEntities = new ArrayList<>();
blobs.iterateAll().forEach(blob -> storageEntities.add(transformBlobToStorageEntity(blob)));
return storageEntities;
}
@Override
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type) {
public List<StorageEntity> listFileStorageEntityRecursively(String resourceAbsolutePath) {
resourceAbsolutePath = transformAbsolutePathToGcsKey(resourceAbsolutePath);
Set<String> visited = new HashSet<>();
List<StorageEntity> storageEntityList = new ArrayList<>();
LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
StorageEntity initialEntity = null;
try {
initialEntity = getFileStatus(path, defaultPath, tenantCode, type);
} catch (Exception e) {
log.error("error while listing files status recursively, path: {}", path, e);
return storageEntityList;
}
foldersToFetch.add(initialEntity);
LinkedList<String> foldersToFetch = new LinkedList<>();
foldersToFetch.addLast(resourceAbsolutePath);
while (!foldersToFetch.isEmpty()) {
String pathToExplore = foldersToFetch.pop().getFullName();
try {
List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
String pathToExplore = foldersToFetch.pop();
visited.add(pathToExplore);
List<StorageEntity> tempList = listStorageEntity(pathToExplore);
for (StorageEntity temp : tempList) {
if (temp.isDirectory()) {
foldersToFetch.add(temp);
if (visited.contains(temp.getFullName())) {
continue;
}
foldersToFetch.add(temp.getFullName());
}
}
storageEntityList.addAll(tempList);
} catch (Exception e) {
log.error("error while listing files stat:wus recursively, path: {}", pathToExplore, e);
}
}
return storageEntityList;
}
@Override
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
List<StorageEntity> storageEntityList = new ArrayList<>();
Page<Blob> blobs;
try {
blobs =
gcsStorage.list(
bucketName,
Storage.BlobListOption.prefix(path),
Storage.BlobListOption.currentDirectory());
} catch (Exception e) {
throw new RuntimeException("Get GCS file list exception. ", e);
public StorageEntity getStorageEntity(String resourceAbsolutePath) {
resourceAbsolutePath = transformAbsolutePathToGcsKey(resourceAbsolutePath);
Blob blob = gcsStorage.get(BlobId.of(bucketName, resourceAbsolutePath));
return transformBlobToStorageEntity(blob);
}
if (blobs == null) {
return storageEntityList;
}
for (Blob blob : blobs.iterateAll()) {
if (path.equals(blob.getName())) {
continue;
}
if (blob.isDirectory()) {
String suffix = StringUtils.difference(path, blob.getName());
String fileName = StringUtils.difference(defaultPath, blob.getName());
StorageEntity entity = new StorageEntity();
entity.setAlias(suffix);
entity.setFileName(fileName);
entity.setFullName(blob.getName());
entity.setDirectory(true);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
entity.setCreateTime(null);
entity.setUpdateTime(null);
entity.setPfullName(path);
storageEntityList.add(entity);
} else {
String[] aliasArr = blob.getName().split("/");
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, blob.getName());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(blob.getName());
entity.setDirectory(false);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(blob.getSize());
entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant()));
entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant()));
entity.setPfullName(path);
storageEntityList.add(entity);
}
}
return storageEntityList;
}
@Override
public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
if (path.endsWith(FOLDER_SEPARATOR)) {
// the path is a directory that may or may not exist
String alias = findDirAlias(path);
String fileName = StringUtils.difference(defaultPath, path);
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(path);
entity.setDirectory(true);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
return entity;
} else {
if (isObjectExists(path)) {
Blob blob = gcsStorage.get(BlobId.of(bucketName, path));
String[] aliasArr = blob.getName().split(FOLDER_SEPARATOR);
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, blob.getName());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(blob.getName());
entity.setDirectory(false);
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(blob.getSize());
entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant()));
entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant()));
return entity;
} else {
throw new FileNotFoundException("Object is not found in GCS Bucket: " + bucketName);
}
}
}
protected boolean isObjectExists(String objectName) {
Blob blob = gcsStorage.get(BlobId.of(bucketName, objectName));
return blob != null && blob.exists();
}
public void checkBucketNameExists(String bucketName) {
private void checkBucketNameExists(String bucketName) {
if (StringUtils.isBlank(bucketName)) {
throw new IllegalArgumentException(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME + " is blank");
}
@ -483,12 +262,35 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
}
}
private String findDirAlias(String dirPath) {
if (!dirPath.endsWith(FOLDER_SEPARATOR)) {
return dirPath;
private StorageEntity transformBlobToStorageEntity(Blob blob) {
String absolutePath = transformGcsKeyToAbsolutePath(blob.getName());
ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath);
StorageEntity entity = new StorageEntity();
entity.setFileName(new File(absolutePath).getName());
entity.setFullName(absolutePath);
entity.setDirectory(resourceMetaData.isDirectory());
entity.setType(resourceMetaData.getResourceType());
entity.setSize(blob.getSize());
entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant()));
entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant()));
return entity;
}
Path path = Paths.get(dirPath);
return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR;
private String transformAbsolutePathToGcsKey(String absolutePath) {
ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath);
if (resourceMetaData.isDirectory()) {
return FileUtils.concatFilePath(absolutePath, "/");
}
return absolutePath;
}
private String transformGcsKeyToAbsolutePath(String gcsKey) {
if (gcsKey.endsWith("/")) {
return gcsKey.substring(0, gcsKey.length() - 1);
}
return gcsKey;
}
}

25
dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java
View File

@ -17,20 +17,29 @@
package org.apache.dolphinscheduler.plugin.storage.gcs;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory;
import org.apache.dolphinscheduler.plugin.storage.api.StorageType;
import com.google.auto.service.AutoService;
@AutoService(StorageOperateFactory.class)
public class GcsStorageOperatorFactory implements StorageOperateFactory {
@AutoService(StorageOperatorFactory.class)
public class GcsStorageOperatorFactory implements StorageOperatorFactory {
@Override
public StorageOperate createStorageOperate() {
GcsStorageOperator gcsStorageOperator = new GcsStorageOperator();
gcsStorageOperator.init();
return gcsStorageOperator;
public StorageOperator createStorageOperate() {
final GcsStorageProperties gcsStorageProperties = getGcsStorageProperties();
return new GcsStorageOperator(gcsStorageProperties);
}
public GcsStorageProperties getGcsStorageProperties() {
return GcsStorageProperties.builder()
.resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"))
.credential(PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_CREDENTIAL))
.bucketName(PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME))
.build();
}
@Override

36
dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageProperties.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.gcs;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class GcsStorageProperties {
private String bucketName;
private String credential;
private String resourceUploadPath;
}

290
dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/test/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorTest.java
View File

@ -1,290 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.gcs;
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
@ExtendWith(MockitoExtension.class)
public class GcsStorageOperatorTest {
private static final String CREDENTIAL_MOCK = "CREDENTIAL_MOCK";
private static final String BUCKET_NAME_MOCK = "BUCKET_NAME_MOCK";
private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK";
private static final String DIR_MOCK = "DIR_MOCK";
private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK";
private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK";
private static final String FULL_NAME = "/tmp/dir1/";
private static final String DEFAULT_PATH = "/tmp/";
@Mock
private Storage gcsStorage;
private GcsStorageOperator gcsStorageOperator;
@BeforeEach
public void setUp() throws Exception {
gcsStorageOperator = Mockito.spy(GcsStorageOperator.class);
Mockito.doReturn(CREDENTIAL_MOCK).when(gcsStorageOperator).readCredentials();
Mockito.doReturn(BUCKET_NAME_MOCK).when(gcsStorageOperator).readBucketName();
Mockito.doReturn(gcsStorage).when(gcsStorageOperator).buildGcsStorage(Mockito.anyString());
Mockito.doNothing().when(gcsStorageOperator).checkBucketNameExists(Mockito.anyString());
gcsStorageOperator.init();
}
@Test
public void testInit() throws Exception {
verify(gcsStorageOperator, times(1)).buildGcsStorage(CREDENTIAL_MOCK);
Assertions.assertEquals(CREDENTIAL_MOCK, gcsStorageOperator.getCredential());
Assertions.assertEquals(BUCKET_NAME_MOCK, gcsStorageOperator.getBucketName());
}
@Test
public void testClose() throws Exception {
doNothing().when(gcsStorage).close();
gcsStorageOperator.close();
verify(gcsStorage, times(1)).close();
}
@Test
public void createTenantResAndUdfDir() throws Exception {
doReturn(DIR_MOCK).when(gcsStorageOperator).getGcsResDir(TENANT_CODE_MOCK);
doReturn(DIR_MOCK).when(gcsStorageOperator).getGcsUdfDir(TENANT_CODE_MOCK);
doReturn(true).when(gcsStorageOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK);
gcsStorageOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK);
verify(gcsStorageOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK);
}
@Test
public void getResDir() {
final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK);
final String dir = gcsStorageOperator.getResDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedResourceDir, dir);
}
@Test
public void getUdfDir() {
final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK);
final String dir = gcsStorageOperator.getUdfDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedUdfDir, dir);
}
@Test
public void mkdirWhenDirExists() {
boolean isSuccess = false;
try {
final String key = DIR_MOCK + FOLDER_SEPARATOR;
Mockito.doReturn(true).when(gcsStorageOperator).isObjectExists(key);
isSuccess = gcsStorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK);
} catch (IOException e) {
Assertions.fail("test failed due to unexpected IO exception");
}
Assertions.assertTrue(isSuccess);
}
@Test
public void mkdirWhenDirNotExists() {
boolean isSuccess = true;
try {
final String key = DIR_MOCK + FOLDER_SEPARATOR;
doReturn(false).when(gcsStorageOperator).isObjectExists(key);
isSuccess = gcsStorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK);
verify(gcsStorage, times(1)).create(Mockito.any(BlobInfo.class), Mockito.any(byte[].class));
} catch (IOException e) {
Assertions.fail("test failed due to unexpected IO exception");
}
Assertions.assertTrue(isSuccess);
}
@Test
public void getResourceFullName() {
final String expectedResourceFullName =
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
final String resourceFullName = gcsStorageOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
Assertions.assertEquals(expectedResourceFullName, resourceFullName);
}
@Test
public void getResourceFileName() {
final String expectedResourceFileName = FILE_NAME_MOCK;
final String resourceFullName =
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
final String resourceFileName = gcsStorageOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName);
Assertions.assertEquals(expectedResourceFileName, resourceFileName);
}
@Test
public void getFileName() {
final String expectedFileName =
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
final String fileName = gcsStorageOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK);
Assertions.assertEquals(expectedFileName, fileName);
}
@Test
public void exists() {
boolean doesExist = false;
doReturn(true).when(gcsStorageOperator).isObjectExists(FILE_NAME_MOCK);
try {
doesExist = gcsStorageOperator.exists(FILE_NAME_MOCK);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
Assertions.assertTrue(doesExist);
}
@Test
public void delete() {
boolean isDeleted = false;
doReturn(true).when(gcsStorage).delete(Mockito.any(BlobId.class));
doReturn(true).when(gcsStorageOperator).isObjectExists(FILE_NAME_MOCK);
try {
isDeleted = gcsStorageOperator.delete(FILE_NAME_MOCK, true);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
Assertions.assertTrue(isDeleted);
verify(gcsStorage, times(1)).delete(Mockito.any(BlobId.class));
}
@Test
public void copy() {
boolean isSuccess = false;
doReturn(null).when(gcsStorage).copy(Mockito.any());
try {
isSuccess = gcsStorageOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
Assertions.assertTrue(isSuccess);
verify(gcsStorage, times(1)).copy(Mockito.any());
}
@Test
public void deleteTenant() {
doNothing().when(gcsStorageOperator).deleteTenantCode(anyString());
try {
gcsStorageOperator.deleteTenant(TENANT_CODE_MOCK);
} catch (Exception e) {
Assertions.fail("unexpected exception caught in unit test");
}
verify(gcsStorageOperator, times(1)).deleteTenantCode(anyString());
}
@Test
public void getGcsResDir() {
final String expectedGcsResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK);
final String gcsResDir = gcsStorageOperator.getGcsResDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedGcsResDir, gcsResDir);
}
@Test
public void getGcsUdfDir() {
final String expectedGcsUdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK);
final String gcsUdfDir = gcsStorageOperator.getGcsUdfDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedGcsUdfDir, gcsUdfDir);
}
@Test
public void getGcsTenantDir() {
final String expectedGcsTenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK);
doReturn(DIR_MOCK).when(gcsStorageOperator).getGcsDataBasePath();
final String gcsTenantDir = gcsStorageOperator.getGcsTenantDir(TENANT_CODE_MOCK);
Assertions.assertEquals(expectedGcsTenantDir, gcsTenantDir);
}
@Test
public void deleteDir() {
doReturn(true).when(gcsStorageOperator).isObjectExists(Mockito.any());
gcsStorageOperator.deleteDirectory(DIR_MOCK);
verify(gcsStorage, times(1)).delete(Mockito.any(BlobId.class));
}
@Test
public void testGetFileStatus() throws Exception {
StorageEntity entity =
gcsStorageOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
Assertions.assertEquals(FULL_NAME, entity.getFullName());
Assertions.assertEquals("dir1/", entity.getFileName());
}
@Test
public void testListFilesStatus() throws Exception {
Mockito.doReturn(null).when(gcsStorage).list(Mockito.any(), Mockito.any(Storage.BlobListOption.class),
Mockito.any(Storage.BlobListOption.class));
List<StorageEntity> result =
gcsStorageOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
verify(gcsStorage, times(1)).list(Mockito.any(), Mockito.any(Storage.BlobListOption.class),
Mockito.any(Storage.BlobListOption.class));
}
@Test
public void testListFilesStatusRecursively() throws Exception {
StorageEntity entity = new StorageEntity();
entity.setFullName(FULL_NAME);
doReturn(entity).when(gcsStorageOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK,
ResourceType.FILE);
doReturn(Collections.EMPTY_LIST).when(gcsStorageOperator).listFilesStatus(anyString(), anyString(), anyString(),
Mockito.any(ResourceType.class));
List<StorageEntity> result =
gcsStorageOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK,
ResourceType.FILE);
Assertions.assertEquals(0, result.size());
}
}

6
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml
View File

@ -228,5 +228,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

Some files were not shown because too many files have changed in this diff Show More