natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Feature] Should not add refresh logic in the e2e case. (#10331) 

* Should not add refresh logic in the e2e case.

* When the returned data from paging is empty, it should be an empty collection.

* resource query fix.

* e2e rerun

* timeZone ut fix

* udf resource check fix

* code deduplication

* rerun
This commit is contained in:
WangJPLeo 2022-06-02 09:48:32 +08:00 committed by GitHub
parent e8ad62bcd0
commit e57c2d1663
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 34 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java
View File

@ -63,25 +63,6 @@ public class ApiFuncIdentificationConstant {
public static final String CALENDAR_MANAGE = "security:calendar:view";
public static final String CARDS_MANAGER = "security:cards:view";
public static final String USER_MANAGER = "security:user:view";
public static final String ALL_ROLES = "security:user:roles";
public static final String USERS_CREATE = "security:user:create";
public static final String USERS_IMPORT = "security:user:import";
public static final String DOWNLOAD_TEMPLATE = "security:user:template";
public static final String USER_UPDATE = "security:user:update";
public static final String USER_ROLE_ASSOCIATEDE = "security:user:role";
public static final String USER_VIEW_PERMISSIONS = "security:user:permission";
public static final String USER_RESET_PASSWORD = "security:user:reset-pwd";
public static final String USER_DELETE = "security:user:delete";
public static final String USER_REVOKE_PROJECT = "security:user:revoke:project";
public static final String USER_GRANT_K8SNAMESPACE = "security:user:grant:k8snamespace";
public static final String USER_GRANT_PROJECT = "security:user:grant:project";
public static final String USER_BATCH_DELETION = "security:user:batch-delete";
public static final String ROLE_MANAGER = "security:role:view";
public static final String ROLE_CREATE = "security:role:create";
public static final String ROLE_RENAME = "security:role:rename";
public static final String ROLE_DELETE = "security:role:delete";
public static final String VIEW_PERMISSION = "security:role:permission-view";
public static final String ASSIGN_PERMISSION = "security:role:permission-assign";
public static final String PROJECT = "project:view";
public static final String PROJECT_CREATE = "project:create";

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java
View File

@ -123,11 +123,11 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
putMsg(result,Status.USER_NO_OPERATION_PERM);
return result;
}
IPage<AlertGroup> alertGroupIPage;
IPage<AlertGroup> alertGroupPage;
PageInfo<AlertGroup> pageInfo = new PageInfo<>(pageNo, pageSize);
Page<AlertGroup> page = new Page<>(pageNo, pageSize);
if (loginUser.getUserType().equals(UserType.ADMIN_USER)) {
alertGroupIPage = alertGroupMapper.queryAlertGroupPage(page, searchVal);
alertGroupPage = alertGroupMapper.queryAlertGroupPage(page, searchVal);
} else {
Set<Integer> ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger);
if (ids.isEmpty()) {
@ -135,10 +135,10 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
putMsg(result, Status.SUCCESS);
return result;
}
alertGroupIPage = alertGroupMapper.queryAlertGroupPageByIds(page, new ArrayList<>(ids), searchVal);
alertGroupPage = alertGroupMapper.queryAlertGroupPageByIds(page, new ArrayList<>(ids), searchVal);
}
pageInfo.setTotal((int) alertGroupIPage.getTotal());
pageInfo.setTotalList(alertGroupIPage.getRecords());
pageInfo.setTotal((int) alertGroupPage.getTotal());
pageInfo.setTotalList(alertGroupPage.getRecords());
result.setData(pageInfo);
putMsg(result, Status.SUCCESS);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
View File

@ -660,7 +660,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
public Map<String, Object> verifyProcessDefinitionName(User loginUser, long projectCode, String name) {
Project project = projectMapper.queryByCode(projectCode);
//check user access for project
Map<String, Object> result = projectService.checkProjectAndAuth(loginUser, project, projectCode,null);
Map<String, Object> result = projectService.checkProjectAndAuth(loginUser, project, projectCode,WORKFLOW_CREATE);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return result;
}

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
View File

@ -118,7 +118,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
@Autowired(required = false)
private StorageOperate storageOperate;
@Autowired
private ResourcePermissionCheckService resourcePermissionCheckService;
@ -352,7 +352,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
MultipartFile file) {
Result<Object> result = new Result<>();
String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPDATE : ApiFuncIdentificationConstant.UDF_UPDATE;
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, checkResourceType(type), funcPermissionKey);
if (!canOperatorPermissions){
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
@ -640,8 +640,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
PageInfo<Resource> pageInfo = new PageInfo<>(pageNo, pageSize);
Set<Integer> resourcesIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID, loginUser.getId(), logger);
Set<Integer> resourcesIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(checkResourceType(type), loginUser.getId(), logger);
if (resourcesIds.isEmpty()) {
result.setData(pageInfo);
putMsg(result, Status.SUCCESS);
@ -767,7 +766,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
Set<Integer> resourceIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID, loginUser.getId(), logger);
Set<Integer> resourceIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(checkResourceType(type), loginUser.getId(), logger);
if (resourceIds.isEmpty()){
result.setData(Collections.emptyList());
putMsg(result, Status.SUCCESS);
@ -813,12 +812,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return resultCheck;
}
String funcPermissionKey = resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_DELETE : ApiFuncIdentificationConstant.UDF_DELETE;
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, checkResourceType(resource.getType()), funcPermissionKey);
if (!canOperatorPermissions){
putMsg(resultCheck, Status.NO_CURRENT_OPERATING_PERMISSION);
return resultCheck;
}
Result<Object> result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
@ -964,7 +963,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW : ApiFuncIdentificationConstant.UDF_FILE_VIEW;
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resource.getId()}, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resource.getId()}, checkResourceType(type), funcPermissionKey);
if (!canOperatorPermissions){
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
@ -988,7 +987,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
String funcPermissionKey = resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW : ApiFuncIdentificationConstant.UDF_FILE_VIEW;
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, checkResourceType(resource.getType()), funcPermissionKey);
if (!canOperatorPermissions){
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
@ -1019,7 +1018,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
String funcPermissionKey = resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW : ApiFuncIdentificationConstant.UDF_FILE_VIEW;
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, checkResourceType(resource.getType()), funcPermissionKey);
if (!canOperatorPermissions){
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
@ -1088,7 +1087,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
@ -1125,7 +1124,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
updateParentResourceSize(resource, resource.getSize());
putMsg(result, Status.SUCCESS);
permissionPostHandle(AuthorizationType.RESOURCE_FILE_ID, loginUser.getId(), Collections.singletonList(resource.getId()), logger);
permissionPostHandle(checkResourceType(resource.getType()), loginUser.getId(), Collections.singletonList(resource.getId()), logger);
Map<String, Object> resultMap = new HashMap<>();
for (Map.Entry<Object, Object> entry : new BeanMap(resource).entrySet()) {
if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) {
@ -1202,7 +1201,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
String funcPermissionKey = resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPDATE : ApiFuncIdentificationConstant.UDF_UPDATE;
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, checkResourceType(resource.getType()), funcPermissionKey);
if (!canOperatorPermissions){
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
@ -1304,7 +1303,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
String funcPermissionKey = resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_DOWNLOAD : ApiFuncIdentificationConstant.UDF_DOWNLOAD;
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, checkResourceType(resource.getType()), funcPermissionKey);
if (!canOperatorPermissions){
logger.error("{}: {}", Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), PropertyUtils.getResUploadStartupState());
throw new ServiceException(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg());
@ -1341,8 +1340,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
logger.error("download resource error, the path is {}, and local filename is {}, the error message is {}", fileName, localFileName, e.getMessage());
throw new ServerException("download the resource file failed ,it may be related to your storage");
}
}
/**
@ -1429,7 +1426,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
putMsg(result, Status.FUNCTION_DISABLED);
return result;
}
List<UdfFunc> udfFuncList;
if (isAdmin(loginUser)) {
// admin gets all udfs except userId
@ -1465,7 +1462,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
Map<String, Object> result = new HashMap<>();
if (!resourcePermissionCheckService.functionDisabled()){
putMsg(result, Status.FUNCTION_DISABLED);
return result;
return result;
}
List<UdfFunc> udfFuncs = udfFunctionMapper.queryAuthedUdfFunc(userId);
result.put(Constants.DATA_LIST, udfFuncs);
@ -1608,4 +1605,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
List<Integer> resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, perm);
return CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourcesMapper.queryResourceListById(resIds);
}
private AuthorizationType checkResourceType(ResourceType type) {
return type.equals(ResourceType.FILE) ? AuthorizationType.RESOURCE_FILE_ID : AuthorizationType.UDF_FILE;
}
}

18
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
View File

@ -157,22 +157,12 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
IPage<Tenant> tenantIPage;
Page<Tenant> page = new Page<>(pageNo, pageSize);
IPage<Tenant> tenantPage = tenantMapper.queryTenantPaging(page, searchVal);
PageInfo<Tenant> pageInfo = new PageInfo<>(pageNo, pageSize);
if (loginUser.getUserType().equals(UserType.ADMIN_USER)) {
tenantIPage = tenantMapper.queryTenantPaging(page, searchVal);
} else {
Set<Integer> ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, loginUser.getId(), logger);
if (ids.isEmpty()) {
result.setData(pageInfo);
putMsg(result, Status.SUCCESS);
return result;
}
tenantIPage = tenantMapper.queryTenantPagingByIds(page, new ArrayList<>(ids), searchVal);
}
pageInfo.setTotal((int) tenantIPage.getTotal());
pageInfo.setTotalList(tenantIPage.getRecords());
pageInfo.setTotal((int) tenantPage.getTotal());
pageInfo.setTotalList(tenantPage.getRecords());
result.setData(pageInfo);
putMsg(result, Status.SUCCESS);
return result;

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java
View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.api.utils;
import java.util.Collections;
import java.util.List;
/**
@ -29,7 +30,7 @@ public class PageInfo<T> {
/**
* totalList
*/
private List<T> totalList;
private List<T> totalList = Collections.emptyList();
/**
* total
*/

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
View File

@ -479,7 +479,7 @@ public class ProcessDefinitionServiceTest {
//project check auth fail
Map<String, Object> result = new HashMap<>();
putMsg(result, Status.PROJECT_NOT_FOUND, projectCode);
Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectCode,null)).thenReturn(result);
Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_CREATE)).thenReturn(result);
Map<String, Object> map = processDefinitionService.verifyProcessDefinitionName(loginUser,
projectCode, "test_pdf");
Assert.assertEquals(Status.PROJECT_NOT_FOUND, map.get(Constants.STATUS));

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
View File

@ -278,8 +278,8 @@ public class ResourcesServiceTest {
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
PowerMockito.when(storageOperate.getFileName(Mockito.any(), Mockito.any(), Mockito.anyString())).thenReturn("test1");
PowerMockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, 1, ApiFuncIdentificationConstant.UDF_UPDATE, serviceLogger)).thenReturn(true);
PowerMockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, new Object[]{1}, 1, serviceLogger)).thenReturn(true);
PowerMockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF_FILE, 1, ApiFuncIdentificationConstant.UDF_UPDATE, serviceLogger)).thenReturn(true);
PowerMockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.UDF_FILE, new Object[]{1}, 1, serviceLogger)).thenReturn(true);
try {
Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(false);
} catch (IOException e) {

1
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ProjectE2ETest.java
View File

@ -55,7 +55,6 @@ class ProjectE2ETest {
@Order(30)
void testDeleteProject() {
final ProjectPage page = new ProjectPage(browser);
browser.navigate().refresh();
page.delete(project);
await().untilAsserted(() -> {

7
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/UdfManageE2ETest.java
View File

@ -120,9 +120,7 @@ public class UdfManageE2ETest {
new WebDriverWait(page.driver(), 10)
.until(ExpectedConditions.urlContains("/resource-manage"));
browser.navigate().refresh();
page.createDirectory(testDirectoryName, "test_desc");
browser.navigate().refresh();
await().untilAsserted(() -> assertThat(page.udfList())
.as("File list should contain newly-created file")
.extracting(WebElement::getText)
@ -151,7 +149,6 @@ public class UdfManageE2ETest {
@Order(30)
void testDeleteDirectory() {
final UdfManagePage page = new UdfManagePage(browser);
browser.navigate().refresh();
page.delete(testDirectoryName);
await().untilAsserted(() -> {
@ -172,9 +169,7 @@ public class UdfManageE2ETest {
final UdfManagePage page = new UdfManagePage(browser);
downloadFile("https://repo1.maven.org/maven2/org/apache/hive/hive-jdbc/3.1.2/hive-jdbc-3.1.2.jar", testUploadUdfFilePath.toFile().getAbsolutePath());
browser.navigate().refresh();
page.uploadFile(testUploadUdfFilePath.toFile().getAbsolutePath());
browser.navigate().refresh();
await().untilAsserted(() -> {
assertThat(page.udfList())
.as("File list should contain newly-created file")
@ -205,7 +200,6 @@ public class UdfManageE2ETest {
@Order(60)
void testRenameUdf() {
final UdfManagePage page = new UdfManagePage(browser);
browser.navigate().refresh();
page.rename(testUploadUdfFileName, testUploadUdfRenameFileName);
await().untilAsserted(() -> {
@ -220,7 +214,6 @@ public class UdfManageE2ETest {
@Order(70)
void testDeleteUdf() {
final UdfManagePage page = new UdfManagePage(browser);
browser.navigate().refresh();
page.delete(testUploadUdfRenameFileName);
await().untilAsserted(() -> {

9
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowE2ETest.java
View File

@ -82,7 +82,6 @@ class WorkflowE2ETest {
@AfterAll
public static void cleanup() {
browser.navigate().refresh();
new NavBarPage(browser)
.goToNav(ProjectPage.class)
.goTo(project)
@ -90,7 +89,6 @@ class WorkflowE2ETest {
.cancelPublishAll()
.deleteAll()
;
browser.navigate().refresh();
new NavBarPage(browser)
.goToNav(ProjectPage.class)
.delete(project)
@ -104,7 +102,6 @@ class WorkflowE2ETest {
@Order(1)
void testCreateWorkflow() {
final String workflow = "test-workflow-1";
browser.navigate().refresh();
WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goTo(project)
@ -131,7 +128,6 @@ class WorkflowE2ETest {
.anyMatch(
it -> it.getText().contains(workflow)
));
browser.navigate().refresh();
workflowDefinitionPage.publish(workflow);
}
@ -139,7 +135,6 @@ class WorkflowE2ETest {
@Order(10)
void testCreateSubWorkflow() {
final String workflow = "test-sub-workflow-1";
browser.navigate().refresh();
WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
@ -164,7 +159,6 @@ class WorkflowE2ETest {
await().untilAsserted(() -> assertThat(
workflowDefinitionPage.workflowList()
).anyMatch(it -> it.getText().contains(workflow)));
browser.navigate().refresh();
workflowDefinitionPage.publish(workflow);
}
@ -172,7 +166,6 @@ class WorkflowE2ETest {
@Order(30)
void testRunWorkflow() {
final String workflow = "test-workflow-1";
browser.navigate().refresh();
final ProjectDetailPage projectPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
@ -181,7 +174,6 @@ class WorkflowE2ETest {
projectPage
.goToTab(WorkflowInstanceTab.class)
.deleteAll();
browser.navigate().refresh();
projectPage
.goToTab(WorkflowDefinitionTab.class)
.run(workflow)
@ -199,7 +191,6 @@ class WorkflowE2ETest {
assertThat(row.isSuccess()).isTrue();
assertThat(row.executionTime()).isEqualTo(1);
});
browser.navigate().refresh();
// Test rerun
projectPage
.goToTab(WorkflowInstanceTab.class)