natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Fix-13434][Resource Center] fix the NPE when use OSS as the resource center (#13435)

This commit is contained in:
Rick Cheng 2023-02-07 11:55:07 +08:00 committed by GitHub
parent 9a6f83383b
commit 8a59ab44eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 248 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
View File

@ -714,6 +714,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
String defaultPath = "";
List<StorageEntity> resourcesList = new ArrayList<>();
String resourceStorageType =
PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE, ResUploadType.NONE.name());
if (isAdmin(loginUser) && StringUtils.isBlank(fullName)) {
// list all tenants' resources to admin users in the root directory
@ -735,7 +737,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
logger.error(e.getMessage() + " Resource path: {}", defaultPath, e);
putMsg(result, Status.RESOURCE_NOT_EXIST);
throw new ServiceException(String.format(e.getMessage() +
" make sure resource path: %s exists in hdfs", defaultPath));
" make sure resource path: %s exists in %s", defaultPath, resourceStorageType));
}
}
}
@ -755,7 +757,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
logger.error(e.getMessage() + " Resource path: {}", fullName, e);
putMsg(result, Status.RESOURCE_NOT_EXIST);
throw new ServiceException(String.format(e.getMessage() +
" make sure resource path: %s exists in hdfs", defaultPath));
" make sure resource path: %s exists in %s", defaultPath, resourceStorageType));
}
}

189
dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
View File

@ -44,7 +44,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -56,8 +60,13 @@ import org.slf4j.LoggerFactory;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSException;
import com.aliyun.oss.ServiceException;
import com.aliyun.oss.model.Bucket;
import com.aliyun.oss.model.DeleteObjectsRequest;
import com.aliyun.oss.model.ListObjectsV2Request;
import com.aliyun.oss.model.ListObjectsV2Result;
import com.aliyun.oss.model.OSSObject;
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.PutObjectRequest;
@ -190,8 +199,20 @@ public class OssStorageOperator implements Closeable, StorageOperate {
}
@Override
public boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException {
return false;
public boolean delete(String fullName, List<String> childrenPathList, boolean recursive) throws IOException {
// append the resource fullName to the list for deletion.
childrenPathList.add(fullName);
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName)
.withKeys(childrenPathList);
try {
ossClient.deleteObjects(deleteObjectsRequest);
} catch (Exception e) {
logger.error("delete objects error", e);
return false;
}
return true;
}
@Override
@ -288,19 +309,168 @@ public class OssStorageOperator implements Closeable, StorageOperate {
@Override
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type) {
return null;
List<StorageEntity> storageEntityList = new ArrayList<>();
LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
StorageEntity initialEntity = null;
try {
initialEntity = getFileStatus(path, defaultPath, tenantCode, type);
} catch (Exception e) {
logger.error("error while listing files status recursively, path: {}", path, e);
return storageEntityList;
}
foldersToFetch.add(initialEntity);
while (!foldersToFetch.isEmpty()) {
String pathToExplore = foldersToFetch.pop().getFullName();
try {
List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
for (StorageEntity temp : tempList) {
if (temp.isDirectory()) {
foldersToFetch.add(temp);
}
}
storageEntityList.addAll(tempList);
} catch (Exception e) {
logger.error("error while listing files status recursively, path: {}", pathToExplore, e);
}
}
return storageEntityList;
}
@Override
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
return null;
List<StorageEntity> storageEntityList = new ArrayList<>();
ListObjectsV2Result result = null;
String nextContinuationToken = null;
do {
try {
ListObjectsV2Request request = new ListObjectsV2Request();
request.setBucketName(bucketName);
request.setPrefix(path);
request.setDelimiter(FOLDER_SEPARATOR);
request.setContinuationToken(nextContinuationToken);
result = ossClient.listObjectsV2(request);
} catch (Exception e) {
throw new ServiceException("Get OSS file list exception", e);
}
List<OSSObjectSummary> summaries = result.getObjectSummaries();
for (OSSObjectSummary summary : summaries) {
if (!summary.getKey().endsWith(FOLDER_SEPARATOR)) {
// the path is a file
String[] aliasArr = summary.getKey().split(FOLDER_SEPARATOR);
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, summary.getKey());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(summary.getKey());
entity.setDirectory(false);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(summary.getSize());
entity.setCreateTime(summary.getLastModified());
entity.setUpdateTime(summary.getLastModified());
entity.setPfullName(path);
storageEntityList.add(entity);
}
}
for (String commonPrefix : result.getCommonPrefixes()) {
// the paths in commonPrefix are directories
String suffix = StringUtils.difference(path, commonPrefix);
String fileName = StringUtils.difference(defaultPath, commonPrefix);
StorageEntity entity = new StorageEntity();
entity.setAlias(suffix);
entity.setFileName(fileName);
entity.setFullName(commonPrefix);
entity.setDirectory(true);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
entity.setCreateTime(null);
entity.setUpdateTime(null);
entity.setPfullName(path);
storageEntityList.add(entity);
}
nextContinuationToken = result.getNextContinuationToken();
} while (result.isTruncated());
return storageEntityList;
}
@Override
public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
ResourceType type) throws Exception {
return null;
ListObjectsV2Request request = new ListObjectsV2Request();
request.setBucketName(bucketName);
request.setPrefix(path);
request.setDelimiter(FOLDER_SEPARATOR);
ListObjectsV2Result result;
try {
result = ossClient.listObjectsV2(request);
} catch (Exception e) {
throw new ServiceException("Get OSS file list exception", e);
}
List<OSSObjectSummary> summaries = result.getObjectSummaries();
if (path.endsWith(FOLDER_SEPARATOR)) {
// the path is a directory that may or may not exist in OSS
String alias = findDirAlias(path);
String fileName = StringUtils.difference(defaultPath, path);
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(path);
entity.setDirectory(true);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(0);
return entity;
} else {
// the path is a file
if (summaries.size() > 0) {
OSSObjectSummary summary = summaries.get(0);
String[] aliasArr = summary.getKey().split(FOLDER_SEPARATOR);
String alias = aliasArr[aliasArr.length - 1];
String fileName = StringUtils.difference(defaultPath, summary.getKey());
StorageEntity entity = new StorageEntity();
entity.setAlias(alias);
entity.setFileName(fileName);
entity.setFullName(summary.getKey());
entity.setDirectory(false);
entity.setDescription("");
entity.setUserName(tenantCode);
entity.setType(type);
entity.setSize(summary.getSize());
entity.setCreateTime(summary.getLastModified());
entity.setUpdateTime(summary.getLastModified());
return entity;
}
}
throw new FileNotFoundException("Object is not found in OSS Bucket: " + bucketName);
}
@Override
@ -360,4 +530,13 @@ public class OssStorageOperator implements Closeable, StorageOperate {
protected OSS buildOssClient() {
return OssClientFactory.buildOssClient(ossConnection);
}
private String findDirAlias(String dirPath) {
if (!dirPath.endsWith(FOLDER_SEPARATOR)) {
return dirPath;
}
Path path = Paths.get(dirPath);
return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR;
}
}

40
dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/test/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorTest.java
View File

@ -27,18 +27,24 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.aliyun.oss.OSS;
import com.aliyun.oss.model.ListObjectsV2Request;
import com.aliyun.oss.model.ListObjectsV2Result;
@ExtendWith(MockitoExtension.class)
public class OssStorageOperatorTest {
@ -53,6 +59,10 @@ public class OssStorageOperatorTest {
private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK";
private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK";
private static final String FULL_NAME = "/tmp/dir1/";
private static final String DEFAULT_PATH = "/tmp/";
@Mock
private OSS ossClientMock;
@ -248,4 +258,34 @@ public class OssStorageOperatorTest {
ossOperator.deleteDir(DIR_MOCK);
verify(ossClientMock, times(1)).deleteObject(anyString(), anyString());
}
@Test
public void testGetFileStatus() throws Exception {
doReturn(new ListObjectsV2Result()).when(ossClientMock).listObjectsV2(Mockito.any(ListObjectsV2Request.class));
StorageEntity entity = ossOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
Assertions.assertEquals(FULL_NAME, entity.getFullName());
Assertions.assertEquals("dir1/", entity.getFileName());
}
@Test
public void testListFilesStatus() throws Exception {
doReturn(new ListObjectsV2Result()).when(ossClientMock).listObjectsV2(Mockito.any(ListObjectsV2Request.class));
List<StorageEntity> result =
ossOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
Assertions.assertEquals(0, result.size());
}
@Test
public void testListFilesStatusRecursively() throws Exception {
StorageEntity entity = new StorageEntity();
entity.setFullName(FULL_NAME);
doReturn(entity).when(ossOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
doReturn(Collections.EMPTY_LIST).when(ossOperator).listFilesStatus(anyString(), anyString(), anyString(),
Mockito.any(ResourceType.class));
List<StorageEntity> result =
ossOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE);
Assertions.assertEquals(0, result.size());
}
}

41
dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
View File

@ -43,6 +43,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -409,34 +411,31 @@ public class S3StorageOperator implements Closeable, StorageOperate {
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
ResourceType type) {
List<StorageEntity> storageEntityList = new ArrayList<>();
LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
do {
String pathToExplore = "";
if (foldersToFetch.size() == 0) {
pathToExplore = path;
} else {
pathToExplore = foldersToFetch.pop().getFullName();
}
StorageEntity initialEntity = null;
try {
initialEntity = getFileStatus(path, defaultPath, tenantCode, type);
} catch (Exception e) {
logger.error("error while listing files status recursively, path: {}", path, e);
return storageEntityList;
}
foldersToFetch.add(initialEntity);
while (!foldersToFetch.isEmpty()) {
String pathToExplore = foldersToFetch.pop().getFullName();
try {
List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
for (StorageEntity temp : tempList) {
if (temp.isDirectory()) {
foldersToFetch.add(temp);
}
}
storageEntityList.addAll(tempList);
} catch (AmazonServiceException e) {
logger.error("Resource path: {}", pathToExplore, e);
// return the resources fetched before error occurs.
return storageEntityList;
} catch (Exception e) {
logger.error("error while listing files status recursively, path: {}", pathToExplore, e);
}
} while (foldersToFetch.size() != 0);
}
return storageEntityList;
@ -585,16 +584,12 @@ public class S3StorageOperator implements Closeable, StorageOperate {
* a directory is a path ending with "/"
*/
private String findDirAlias(String myStr) {
if (!myStr.endsWith("/")) {
if (!myStr.endsWith(FOLDER_SEPARATOR)) {
// Make sure system won't crush down if someone accidentally misuse the function.
return myStr;
}
int lastIndex = myStr.lastIndexOf("/");
String subbedString = myStr.substring(0, lastIndex);
int secondLastIndex = subbedString.lastIndexOf("/");
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(myStr, secondLastIndex + 1, lastIndex + 1);
return stringBuilder.toString();
Path path = Paths.get(myStr);
return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR;
}
}

2
dolphinscheduler-ui/src/locales/zh_CN/resource.ts
View File

@ -24,6 +24,7 @@ export default {
enter_keyword_tips: '请输入关键词',
name: '名称',
user_name: '所属用户',
tenant_name: '所属租户',
whether_directory: '是否文件夹',
file_name: '文件名称',
description: '描述',
@ -58,6 +59,7 @@ export default {
upload_udf_resources: '上传UDF资源',
udf_source_name: 'UDF资源名称',
user_name: '所属用户',
tenant_name: '所属租户',
whether_directory: '是否文件夹',
file_name: '文件名称',
file_size: '文件大小',