natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Fix-1360]: Fix Description length check (#11371) 

* [Fix-1360]: Fix Description length check

* Update dolphinscheduler_h2.sql

h2 Increase the length of some fields

* Update dolphinscheduler_ddl.sql
This commit is contained in:
insist777 2022-08-11 13:26:50 +08:00 committed by GitHub
parent de4e4af844
commit 8217784044
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 149 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
View File

@ -436,6 +436,7 @@ public enum Status {
NO_CURRENT_OPERATING_PERMISSION(1400001, "The current user does not have this permission.", "当前用户无此权限"),
FUNCTION_DISABLED(1400002, "The current feature is disabled.", "当前功能已被禁用"),
SCHEDULE_TIME_NUMBER(1400003, "The number of complement dates exceed 100.", "补数日期个数超过100"),
DESCRIPTION_TOO_LONG_ERROR(1400004, "description is too long error", "描述过长"),
;
private final int code;

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java
View File

@ -114,4 +114,13 @@ public interface BaseService {
* @return map<status,startDate,endDate>
*/
Map<String, Object> checkAndParseDateParameters(String startDateStr, String endDateStr);
/**
* check checkDescriptionLength
*
* @param description input String
* @return ture if Length acceptable, Length exceeds return false
*/
boolean checkDescriptionLength(String description);
}

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java
View File

@ -168,7 +168,10 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
AlertGroup alertGroup = new AlertGroup();
Date now = new Date();
@ -215,7 +218,10 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
AlertGroup alertGroup = alertGroupMapper.selectById(id);
if (alertGroup == null) {

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java
View File

@ -212,4 +212,9 @@ public class BaseServiceImpl implements BaseService {
putMsg(result, Status.SUCCESS);
return result;
}
@Override
public boolean checkDescriptionLength(String description) {
return description!=null && description.codePointCount(0, description.length()) > 255;
}
}

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java
View File

@ -283,6 +283,11 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
Map<String, Object> checkResult = checkParams(name, config);
if (checkResult.get(Constants.STATUS) != Status.SUCCESS) {
return checkResult;

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
View File

@ -114,6 +114,10 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
putMsg(result, Status.DATASOURCE_EXIST);
return result;
}
if(checkDescriptionLength(datasourceParam.getNote())){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
// check connect
ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(datasourceParam);
Result<Object> isConnection = checkConnection(datasourceParam.getType(), connectionParam);
@ -174,6 +178,10 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
putMsg(result, Status.DATASOURCE_EXIST);
return result;
}
if(checkDescriptionLength(dataSourceParam.getNote())){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
//check password，if the password is not updated, set to the old password.
BaseConnectionParam connectionParam = (BaseConnectionParam) DataSourceUtils.buildConnectionParams(dataSourceParam);
String password = connectionParam.getPassword();

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
View File

@ -100,7 +100,10 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
Map<String, Object> checkResult = checkParams(name,config,workerGroups);
if (checkResult.get(Constants.STATUS) != Status.SUCCESS) {
return checkResult;
@ -355,6 +358,10 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
if (checkResult.get(Constants.STATUS) != Status.SUCCESS) {
return checkResult;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
Environment environment = environmentMapper.queryByEnvironmentName(name);
if (environment != null && !environment.getCode().equals(code)) {

20
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
View File

@ -250,7 +250,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return result;
}
if(checkDescriptionLength(description)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
// check whether the new process define name exist
ProcessDefinition definition = processDefinitionMapper.verifyByDefineName(project.getCode(), name);
if (definition != null) {
@ -578,7 +581,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return result;
}
if(checkDescriptionLength(description)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
List<TaskDefinitionLog> taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class);
Map<String, Object> checkTaskDefinitions = checkTaskDefinitionList(taskDefinitionLogs, taskDefinitionJson);
if (checkTaskDefinitions.get(Constants.STATUS) != Status.SUCCESS) {
@ -2078,7 +2084,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return result;
}
if(checkDescriptionLength(description)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
// check whether the new process define name exist
ProcessDefinition definition = processDefinitionMapper.verifyByDefineName(project.getCode(), name);
if (definition != null) {
@ -2207,7 +2216,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return result;
}
if(checkDescriptionLength(description)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
int tenantId = -1;
if (!Constants.DEFAULT.equals(tenantCode)) {
Tenant tenant = tenantMapper.queryByTenantCode(tenantCode);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java
View File

@ -146,7 +146,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
* @param desc desc
*/
public static void checkDesc(Result result, String desc) {
if (!StringUtils.isEmpty(desc) && desc.length() > 200) {
if (!StringUtils.isEmpty(desc) && desc.codePointCount(0, desc.length()) > 255) {
result.setCode(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode());
result.setMsg(MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "desc length"));
} else {

18
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
View File

@ -159,6 +159,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
return result;
}
if(checkDescriptionLength(description)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
String fullName = getFullName(currentDir, name);
result = verifyResource(loginUser, type, fullName, pid);
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
@ -240,6 +246,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
// make sure login user has tenant
String tenantCode = getTenantCode(loginUser.getId(), result);
@ -369,6 +379,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
if (!PropertyUtils.getResUploadStartupState()) {
putMsg(result, Status.STORAGE_NOT_STARTUP);
@ -1072,6 +1086,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
//check file suffix
String nameSuffix = fileSuffix.trim();

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java
View File

@ -84,12 +84,15 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe
@Transactional
public Map<String, Object> createTaskGroup(User loginUser, Long projectCode, String name, String description, int groupSize) {
Map<String, Object> result = new HashMap<>();
boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.TASK_GROUP, ApiFuncIdentificationConstant.TASK_GROUP_CREATE);
if (!canOperatorPermissions){
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
if(checkDescriptionLength(description)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
if (name == null) {
putMsg(result, Status.NAME_NULL);
return result;
@ -136,6 +139,10 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
if(checkDescriptionLength(description)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
if (name == null) {
putMsg(result, Status.NAME_NULL);
return result;

11
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
View File

@ -146,7 +146,10 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT, TENANT_CREATE)) {
throw new ServiceException(Status.USER_NO_OPERATION_PERM);
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
Tenant tenant = new Tenant(tenantCode, desc, queueId);
createTenantValid(tenant);
tenantMapper.insert(tenant);
@ -211,7 +214,10 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT,TENANT_UPDATE)) {
throw new ServiceException(Status.USER_NO_OPERATION_PERM);
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
Tenant updateTenant = new Tenant(id, tenantCode, desc, queueId);
Tenant existsTenant = tenantMapper.queryById(id);
updateTenantValid(existsTenant, updateTenant);
@ -365,7 +371,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
if (checkTenantExists(tenantCode)) {
return tenantMapper.queryByTenantCode(tenantCode);
}
Queue queueObj = queueService.createQueueIfNotExists(queue, queueName);
Tenant tenant = new Tenant(tenantCode, desc, queueObj.getId());
createTenantValid(tenant);

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
View File

@ -93,6 +93,10 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
// if resource upload startup
if (!PropertyUtils.getResUploadStartupState()) {
logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState());
@ -203,7 +207,10 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
if(checkDescriptionLength(desc)){
putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
return result;
}
// verify udfFunc is exist
UdfFunc udf = udfFuncMapper.selectUdfById(udfFuncId);

18
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
View File

@ -17,7 +17,10 @@
package org.apache.dolphinscheduler.api.service;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.enums.Status;
@ -27,15 +30,11 @@ import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.storage.StorageOperate;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
@ -253,7 +252,11 @@ public class ResourcesServiceTest {
result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
//Description_Lingth_ERROR
result = resourcesService.createDirectory(user, "directoryTest", "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",
ResourceType.FILE, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.DESCRIPTION_TOO_LONG_ERROR.getMsg(), result.getMsg());
}
@Test
@ -981,7 +984,6 @@ public class ResourcesServiceTest {
logger.error("hadoop error", e);
}
}
private List<Resource> getResourceList() {
List<Resource> resources = new ArrayList<>();

4
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
View File

@ -635,7 +635,7 @@ CREATE TABLE t_ds_project
id int(11) NOT NULL AUTO_INCREMENT,
name varchar(100) DEFAULT NULL,
code bigint(20) NOT NULL,
description varchar(200) DEFAULT NULL,
description varchar(255) DEFAULT NULL,
user_id int(11) DEFAULT NULL,
flag tinyint(4) DEFAULT '1',
create_time datetime NOT NULL,
@ -1868,7 +1868,7 @@ CREATE TABLE t_ds_task_group
(
id int(11) NOT NULL AUTO_INCREMENT ,
name varchar(100) DEFAULT NULL ,
description varchar(200) DEFAULT NULL ,
description varchar(255) DEFAULT NULL ,
group_size int(11) NOT NULL ,
project_code bigint(20) DEFAULT '0',
use_size int(11) DEFAULT '0' ,

4
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
View File

@ -638,7 +638,7 @@ CREATE TABLE `t_ds_project` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
`name` varchar(100) DEFAULT NULL COMMENT 'project name',
`code` bigint(20) NOT NULL COMMENT 'encoding',
`description` varchar(200) DEFAULT NULL,
`description` varchar(255) DEFAULT NULL,
`user_id` int(11) DEFAULT NULL COMMENT 'creator id',
`flag` tinyint(4) DEFAULT '1' COMMENT '0 not available, 1 available',
`create_time` datetime NOT NULL COMMENT 'create time',
@ -1859,7 +1859,7 @@ DROP TABLE IF EXISTS `t_ds_task_group`;
CREATE TABLE `t_ds_task_group` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT'key',
`name` varchar(100) DEFAULT NULL COMMENT 'task_group name',
`description` varchar(200) DEFAULT NULL,
`description` varchar(255) DEFAULT NULL,
`group_size` int (11) NOT NULL COMMENT'group size',
`use_size` int (11) DEFAULT '0' COMMENT 'used size',
`user_id` int(11) DEFAULT NULL COMMENT 'creator id',

4
dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
View File

@ -564,7 +564,7 @@ CREATE TABLE t_ds_project (
id int NOT NULL ,
name varchar(100) DEFAULT NULL ,
code bigint NOT NULL,
description varchar(200) DEFAULT NULL ,
description varchar(255) DEFAULT NULL ,
user_id int DEFAULT NULL ,
flag int DEFAULT '1' ,
create_time timestamp DEFAULT CURRENT_TIMESTAMP ,
@ -1856,7 +1856,7 @@ DROP TABLE IF EXISTS t_ds_task_group;
CREATE TABLE t_ds_task_group (
id serial NOT NULL,
name varchar(100) DEFAULT NULL ,
description varchar(200) DEFAULT NULL ,
description varchar(255) DEFAULT NULL ,
group_size int NOT NULL ,
project_code bigint DEFAULT '0' ,
use_size int DEFAULT '0' ,

6
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.0_schema/mysql/dolphinscheduler_ddl.sql
View File

@ -15,7 +15,13 @@
* limitations under the License.
*/
ALTER TABLE `t_ds_task_definition` ADD COLUMN `task_execute_type` int(11) DEFAULT '0' COMMENT 'task execute type: 0-batch, 1-stream' AFTER `task_type`;
ALTER TABLE `t_ds_task_definition_log` ADD COLUMN `task_execute_type` int(11) DEFAULT '0' COMMENT 'task execute type: 0-batch, 1-stream' AFTER `task_type`;
ALTER TABLE `t_ds_task_instance` ADD COLUMN `task_execute_type` int(11) DEFAULT '0' COMMENT 'task execute type: 0-batch, 1-stream' AFTER `task_type`;
ALTER TABLE `t_ds_task_instance` DROP FOREIGN KEY foreign_key_instance_id;
SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));
alter table `t_ds_project` modify `description` varchar(255);
alter table `t_ds_task_group` modify `description` varchar(255);

11
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.0_schema/postgresql/dolphinscheduler_ddl.sql
View File

@ -16,6 +16,9 @@
*/
delimiter d//
CREATE OR REPLACE FUNCTION public.dolphin_update_metadata(
)
RETURNS character varying
@ -28,6 +31,9 @@ v_schema varchar;
BEGIN
---get schema name
v_schema =current_schema();
ALTER TABLE t_ds_project alter COLUMN description type varchar(255);
ALTER TABLE t_ds_task_group alter COLUMN description type varchar(255);
--- add column
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_task_definition ADD COLUMN IF NOT EXISTS task_execute_type int DEFAULT ''0'' ';
@ -40,10 +46,13 @@ EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_task_instance DROP CONS
return 'Success!';
exception when others then
---Raise EXCEPTION '(%)',SQLERRM;
return SQLERRM;
END;
$BODY$;
select dolphin_update_metadata();
d//
d//

14
dolphinscheduler-dao/src/main/resources/sql/upgrade/3.1.0_schema/postgresql/dolphinscheduler_dml.sql
View File

@ -14,3 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
delimiter d//
return 'Success!';
exception when others then
---Raise EXCEPTION '(%)',SQLERRM;
return SQLERRM;
END;
$BODY$;
select dolphin_insert_dq_initial_data();
d//