diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java index a31a423b85..c6215caa23 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.controller; import static org.apache.dolphinscheduler.api.enums.Status.CREATE_ALERT_GROUP_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.DELETE_ALERT_GROUP_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.LIST_PAGING_ALERT_GROUP_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALERT_GROUP_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ALL_ALERTGROUP_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ALERT_GROUP_ERROR; @@ -143,6 +144,7 @@ public class AlertGroupController extends BaseController { return alertGroupService.listPaging(loginUser, searchVal, pageNo, pageSize); } + /** * updateProcessInstance alert group * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java index 424f4aba30..326c7f44b4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java @@ -169,5 +169,4 @@ public class DataAnalysisController extends BaseController { Map result = dataAnalysisService.countQueueState(loginUser); return returnDataList(result); } - } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index b17fd69aef..041af44434 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -143,7 +143,7 @@ public class DataSourceController extends BaseController { } /** - * query datasouce by type + * query datasource by type * * @param loginUser login user * @param type data source type diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java new file mode 100644 index 0000000000..79bebb745f --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.controller; + +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_ENVIRONMENT_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_ENVIRONMENT_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ENVIRONMENT_BY_CODE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ENVIRONMENT_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ENVIRONMENT_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_ENVIRONMENT_ERROR; + +import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; +import org.apache.dolphinscheduler.api.exceptions.ApiException; +import org.apache.dolphinscheduler.api.service.EnvironmentService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import springfox.documentation.annotations.ApiIgnore; + +/** + * environment controller + */ +@Api(tags = "ENVIRONMENT_TAG") +@RestController +@RequestMapping("environment") +public class EnvironmentController extends BaseController { + + @Autowired + private EnvironmentService environmentService; + + /** + * create environment + * + * @param loginUser login user + * @param name environment name + * @param config config + * @param description description + * @return returns an error if it exists + */ + @ApiOperation(value = "createEnvironment", notes = "CREATE_ENVIRONMENT_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "name", value = "ENVIRONMENT_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "config", value = "CONFIG", required = true, dataType = "String"), + @ApiImplicitParam(name = "description", value = "ENVIRONMENT_DESC", dataType = "String"), + @ApiImplicitParam(name = "workerGroups", value = "WORKER_GROUP_LIST", dataType = "String") + }) + @PostMapping(value = "/create") + @ResponseStatus(HttpStatus.CREATED) + @ApiException(CREATE_ENVIRONMENT_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result createProject(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam("name") String name, + @RequestParam("config") String config, + @RequestParam(value = "description", required = false) String description, + @RequestParam(value = "workerGroups", required = false) String workerGroups) { + + Map result = environmentService.createEnvironment(loginUser, name, config, description, workerGroups); + return returnDataList(result); + } + + /** + * update environment + * + * @param loginUser login user + * @param code environment code + * @param name environment name + * @param config environment config + * @param description description + * @return update result code + */ + @ApiOperation(value = "updateEnvironment", notes = "UPDATE_ENVIRONMENT_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "code", value = "ENVIRONMENT_CODE", required = true, dataType = "Long", example = "100"), + @ApiImplicitParam(name = "name", value = "ENVIRONMENT_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "config", value = "ENVIRONMENT_CONFIG", required = true, dataType = "String"), + @ApiImplicitParam(name = "description", value = "ENVIRONMENT_DESC", dataType = "String"), + @ApiImplicitParam(name = "workerGroups", value = "WORKER_GROUP_LIST", dataType = "String") + }) + @PostMapping(value = "/update") + @ResponseStatus(HttpStatus.OK) + @ApiException(UPDATE_ENVIRONMENT_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result updateEnvironment(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam("code") Long code, + @RequestParam("name") String name, + @RequestParam("config") String config, + @RequestParam(value = "description", required = false) String description, + @RequestParam(value = "workerGroups", required = false) String workerGroups) { + Map result = environmentService.updateEnvironmentByCode(loginUser, code, name, config, description, workerGroups); + return returnDataList(result); + } + + /** + * query environment details by code + * + * @param environmentCode environment code + * @return environment detail information + */ + @ApiOperation(value = "queryEnvironmentByCode", notes = "QUERY_ENVIRONMENT_BY_CODE_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", required = true, dataType = "Long", example = "100") + }) + @GetMapping(value = "/query-by-code") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_ENVIRONMENT_BY_CODE_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result queryEnvironmentByCode(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam("environmentCode") Long environmentCode) { + + Map result = environmentService.queryEnvironmentByCode(environmentCode); + return returnDataList(result); + } + + /** + * query environment list paging + * + * @param searchVal search value + * @param pageSize page size + * @param pageNo page number + * @return environment list which the login user have permission to see + */ + @ApiOperation(value = "queryEnvironmentListPaging", notes = "QUERY_ENVIRONMENT_LIST_PAGING_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataType = "String"), + @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataType = "Int", example = "20"), + @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataType = "Int", example = "1") + }) + @GetMapping(value = "/list-paging") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_ENVIRONMENT_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result queryEnvironmentListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "searchVal", required = false) String searchVal, + @RequestParam("pageSize") Integer pageSize, + @RequestParam("pageNo") Integer pageNo + ) { + + Result result = checkPageParams(pageNo, pageSize); + if (!result.checkResult()) { + return result; + } + searchVal = ParameterUtils.handleEscapes(searchVal); + result = environmentService.queryEnvironmentListPaging(pageNo, pageSize, searchVal); + return result; + } + + /** + * delete environment by code + * + * @param loginUser login user + * @param environmentCode environment code + * @return delete result code + */ + @ApiOperation(value = "deleteEnvironmentByCode", notes = "DELETE_ENVIRONMENT_BY_CODE_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", required = true, dataType = "Long", example = "100") + }) + @PostMapping(value = "/delete") + @ResponseStatus(HttpStatus.OK) + @ApiException(DELETE_ENVIRONMENT_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result deleteEnvironment(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam("environmentCode") Long environmentCode + ) { + + Map result = environmentService.deleteEnvironmentByCode(loginUser, environmentCode); + return returnDataList(result); + } + + /** + * query all environment list + * + * @param loginUser login user + * @return all environment list + */ + @ApiOperation(value = "queryAllEnvironmentList", notes = "QUERY_ALL_ENVIRONMENT_LIST_NOTES") + @GetMapping(value = "/query-environment-list") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_ENVIRONMENT_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result queryAllEnvironmentList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { + Map result = environmentService.queryAllEnvironmentList(); + return returnDataList(result); + } + + /** + * verify environment and environment name + * + * @param loginUser login user + * @param environmentName environment name + * @return true if the environment name not exists, otherwise return false + */ + @ApiOperation(value = "verifyEnvironment", notes = "VERIFY_ENVIRONMENT_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "environmentName", value = "ENVIRONMENT_NAME", required = true, dataType = "String") + }) + @PostMapping(value = "/verify-environment") + @ResponseStatus(HttpStatus.OK) + @ApiException(VERIFY_ENVIRONMENT_ERROR) + @AccessLogAnnotation(ignoreRequestArgs = "loginUser") + public Result verifyEnvironment(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "environmentName") String environmentName + ) { + Map result = environmentService.verifyEnvironment(environmentName); + return returnDataList(result); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index f99b774b07..93a31f2e8c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -83,6 +83,7 @@ public class ExecutorController extends BaseController { * @param processInstancePriority process instance priority * @param workerGroup worker group * @param timeout timeout + * @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode * @return start process result code */ @ApiOperation(value = "startProcessInstance", notes = "RUN_PROCESS_INSTANCE_NOTES") @@ -98,6 +99,7 @@ public class ExecutorController extends BaseController { @ApiImplicitParam(name = "runMode", value = "RUN_MODE", dataType = "RunMode"), @ApiImplicitParam(name = "processInstancePriority", value = "PROCESS_INSTANCE_PRIORITY", required = true, dataType = "Priority"), @ApiImplicitParam(name = "workerGroup", value = "WORKER_GROUP", dataType = "String", example = "default"), + @ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long", example = "default"), @ApiImplicitParam(name = "timeout", value = "TIMEOUT", dataType = "Int", example = "100"), @ApiImplicitParam(name = "expectedParallelismNumber", value = "EXPECTED_PARALLELISM_NUMBER", dataType = "Int", example = "8") }) @@ -118,9 +120,10 @@ public class ExecutorController extends BaseController { @RequestParam(value = "runMode", required = false) RunMode runMode, @RequestParam(value = "processInstancePriority", required = false) Priority processInstancePriority, @RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup, + @RequestParam(value = "environmentCode", required = false, defaultValue = "-1") Long environmentCode, @RequestParam(value = "timeout", required = false) Integer timeout, @RequestParam(value = "startParams", required = false) String startParams, - @RequestParam(value = "timeout", required = false) Integer expectedParallelismNumber) { + @RequestParam(value = "expectedParallelismNumber", required = false) Integer expectedParallelismNumber) { if (timeout == null) { timeout = Constants.MAX_TASK_TIMEOUT; @@ -130,7 +133,7 @@ public class ExecutorController extends BaseController { startParamMap = JSONUtils.toMap(startParams); } Map result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, execType, failureStrategy, - startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, workerGroup, timeout, startParamMap, expectedParallelismNumber); + startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, workerGroup, environmentCode,timeout, startParamMap, expectedParallelismNumber); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index 9093fbb23a..4f804ef0aa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -106,7 +106,7 @@ public class ProcessDefinitionController extends BaseController { * @param taskDefinitionJson taskDefinitionJson * @return create result code */ - @ApiOperation(value = "save", notes = "CREATE_PROCESS_DEFINITION_NOTES") + @ApiOperation(value = "createProcessDefinition", notes = "CREATE_PROCESS_DEFINITION_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "name", value = "PROCESS_DEFINITION_NAME", required = true, type = "String"), @ApiImplicitParam(name = "locations", value = "PROCESS_DEFINITION_LOCATIONS", required = true, type = "String"), @@ -283,6 +283,7 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectCode", value = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam(value = "pageNo") int pageNo, @RequestParam(value = "pageSize") int pageSize, +<<<<<<< HEAD @PathVariable(value = "code") long code) { Result result = checkPageParams(pageNo, pageSize); @@ -291,6 +292,16 @@ public class ProcessDefinitionController extends BaseController { } result = processDefinitionService.queryProcessDefinitionVersions(loginUser, projectCode, pageNo, pageSize, code); +======= + @RequestParam(value = "processDefinitionCode") long processDefinitionCode) { + Result result = checkPageParams(pageNo, pageSize); + if (!result.checkResult()) { + return result; + } + result = processDefinitionService.queryProcessDefinitionVersions(loginUser + , projectName, pageNo, pageSize, processDefinitionCode); + +>>>>>>> upstream/dev return result; } @@ -374,7 +385,11 @@ public class ProcessDefinitionController extends BaseController { } /** +<<<<<<< HEAD * query detail of process definition by code +======= + * query detail of process definition by id +>>>>>>> upstream/dev * * @param loginUser login user * @param projectCode project code @@ -470,8 +485,13 @@ public class ProcessDefinitionController extends BaseController { return result; } searchVal = ParameterUtils.handleEscapes(searchVal); +<<<<<<< HEAD return processDefinitionService.queryProcessDefinitionListPaging(loginUser, projectCode, searchVal, userId, pageNo, pageSize); +======= + return processDefinitionService.queryProcessDefinitionListPaging(loginUser, projectName, searchVal, pageNo, pageSize, userId); + +>>>>>>> upstream/dev } /** @@ -554,7 +574,11 @@ public class ProcessDefinitionController extends BaseController { */ @ApiOperation(value = "deleteByCode", notes = "DELETE_PROCESS_DEFINITION_BY_ID_NOTES") @ApiImplicitParams({ +<<<<<<< HEAD @ApiImplicitParam(name = "code", value = "PROCESS_DEFINITION_CODE", dataType = "Int", example = "100") +======= + @ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100") +>>>>>>> upstream/dev }) @DeleteMapping(value = "/{code}") @ResponseStatus(HttpStatus.OK) @@ -577,7 +601,11 @@ public class ProcessDefinitionController extends BaseController { */ @ApiOperation(value = "batchDeleteByCodes", notes = "BATCH_DELETE_PROCESS_DEFINITION_BY_IDS_NOTES") @ApiImplicitParams({ +<<<<<<< HEAD @ApiImplicitParam(name = "codes", value = "PROCESS_DEFINITION_CODE", required = true, dataType = "String") +======= + @ApiImplicitParam(name = "processDefinitionIds", value = "PROCESS_DEFINITION_IDS", required = true, type = "String") +>>>>>>> upstream/dev }) @PostMapping(value = "/batch-delete") @ResponseStatus(HttpStatus.OK) @@ -645,8 +673,16 @@ public class ProcessDefinitionController extends BaseController { * @param projectCode project code * @return process definition list */ +<<<<<<< HEAD @ApiOperation(value = "queryAllByProjectCode", notes = "QUERY_PROCESS_DEFINITION_All_BY_PROJECT_CODE_NOTES") @GetMapping(value = "/all") +======= + @ApiOperation(value = "queryProcessDefinitionAllByProjectId", notes = "QUERY_PROCESS_DEFINITION_All_BY_PROJECT_ID_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "projectId", value = "PROJECT_ID", required = true, dataType = "Int", example = "100") + }) + @GetMapping(value = "/queryProcessDefinitionAllByProjectId") +>>>>>>> upstream/dev @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_PROCESS_DEFINITION_LIST) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index ef591870b7..ca3bd5f563 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -216,7 +216,6 @@ public class ProjectController extends BaseController { return returnDataList(result); } - /** * query authorized project * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index d5fa60f531..b36e116e27 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -474,6 +474,7 @@ public class ResourcesController extends BaseController { */ @ApiOperation(value = "createUdfFunc", notes = "CREATE_UDF_FUNCTION_NOTES") @ApiImplicitParams({ +<<<<<<< HEAD @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType"), @ApiImplicitParam(name = "funcName", value = "FUNC_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "className", value = "CLASS_NAME", required = true, dataType = "String"), @@ -481,6 +482,15 @@ public class ResourcesController extends BaseController { @ApiImplicitParam(name = "database", value = "DATABASE_NAME", dataType = "String"), @ApiImplicitParam(name = "description", value = "UDF_DESC", dataType = "String"), @ApiImplicitParam(name = "resourceId", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100") +======= + @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType"), + @ApiImplicitParam(name = "funcName", value = "FUNC_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "className", value = "CLASS_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "argTypes", value = "ARG_TYPES", dataType = "String"), + @ApiImplicitParam(name = "database", value = "DATABASE_NAME", dataType = "String"), + @ApiImplicitParam(name = "description", value = "UDF_DESC", dataType = "String"), + @ApiImplicitParam(name = "resourceId", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100") +>>>>>>> upstream/dev }) @PostMapping(value = "/{resourceId}/udf-func") @@ -573,9 +583,15 @@ public class ResourcesController extends BaseController { */ @ApiOperation(value = "queryUdfFuncListPaging", notes = "QUERY_UDF_FUNCTION_LIST_PAGING_NOTES") @ApiImplicitParams({ +<<<<<<< HEAD @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataType = "String"), @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataType = "Int", example = "1"), @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataType = "Int", example = "20") +======= + @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataType = "String"), + @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataType = "Int", example = "1"), + @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataType = "Int", example = "20") +>>>>>>> upstream/dev }) @GetMapping(value = "/udf-func") @ResponseStatus(HttpStatus.OK) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index 3b40fd64c2..f3b5290d65 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -99,6 +99,7 @@ public class SchedulerController extends BaseController { @ApiImplicitParam(name = "warningGroupId", value = "WARNING_GROUP_ID", dataType = "Int", example = "100"), @ApiImplicitParam(name = "failureStrategy", value = "FAILURE_STRATEGY", type = "FailureStrategy"), @ApiImplicitParam(name = "workerGroupId", value = "WORKER_GROUP_ID", dataType = "Int", example = "100"), + @ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long"), @ApiImplicitParam(name = "processInstancePriority", value = "PROCESS_INSTANCE_PRIORITY", type = "Priority"), }) @PostMapping() @@ -113,9 +114,10 @@ public class SchedulerController extends BaseController { @RequestParam(value = "warningGroupId", required = false, defaultValue = DEFAULT_NOTIFY_GROUP_ID) int warningGroupId, @RequestParam(value = "failureStrategy", required = false, defaultValue = DEFAULT_FAILURE_POLICY) FailureStrategy failureStrategy, @RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup, + @RequestParam(value = "environmentCode", required = false, defaultValue = "-1") Long environmentCode, @RequestParam(value = "processInstancePriority", required = false, defaultValue = DEFAULT_PROCESS_INSTANCE_PRIORITY) Priority processInstancePriority) { Map result = schedulerService.insertSchedule(loginUser, projectCode, processDefinitionCode, schedule, - warningType, warningGroupId, failureStrategy, processInstancePriority, workerGroup); + warningType, warningGroupId, failureStrategy, processInstancePriority, workerGroup, environmentCode); return returnDataList(result); } @@ -143,6 +145,7 @@ public class SchedulerController extends BaseController { @ApiImplicitParam(name = "failureStrategy", value = "FAILURE_STRATEGY", type = "FailureStrategy"), @ApiImplicitParam(name = "workerGroupId", value = "WORKER_GROUP_ID", dataType = "Int", example = "100"), @ApiImplicitParam(name = "processInstancePriority", value = "PROCESS_INSTANCE_PRIORITY", type = "Priority"), + @ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long"), }) @PutMapping("/{id}") @ApiException(UPDATE_SCHEDULE_ERROR) @@ -155,10 +158,11 @@ public class SchedulerController extends BaseController { @RequestParam(value = "warningGroupId", required = false) int warningGroupId, @RequestParam(value = "failureStrategy", required = false, defaultValue = "END") FailureStrategy failureStrategy, @RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup, + @RequestParam(value = "environmentCode", required = false, defaultValue = "-1") Long environmentCode, @RequestParam(value = "processInstancePriority", required = false) Priority processInstancePriority) { Map result = schedulerService.updateSchedule(loginUser, projectCode, id, schedule, - warningType, warningGroupId, failureStrategy, processInstancePriority, workerGroup); + warningType, warningGroupId, failureStrategy, processInstancePriority, workerGroup, environmentCode); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 19f9c1a510..35ad434f4e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -107,7 +107,6 @@ public class TaskInstanceController extends BaseController { @RequestParam(value = "endDate", required = false) String endTime, @RequestParam("pageNo") Integer pageNo, @RequestParam("pageSize") Integer pageSize) { - Result result = checkPageParams(pageNo, pageSize); if (!result.checkResult()) { return result; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java index 25cd5104d0..6d1ea6b218 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java @@ -144,11 +144,11 @@ public class TenantController extends BaseController { /** - * udpate tenant + * update tenant * * @param loginUser login user - * @param id tennat id - * @param tenantCode tennat code + * @param id tenant id + * @param tenantCode tenant code * @param queueId queue id * @param description description * @return update result code @@ -159,7 +159,6 @@ public class TenantController extends BaseController { @ApiImplicitParam(name = "tenantCode", value = "TENANT_CODE", required = true, dataType = "String"), @ApiImplicitParam(name = "queueId", value = "QUEUE_ID", required = true, dataType = "Int", example = "100"), @ApiImplicitParam(name = "description", value = "TENANT_DESC", type = "String") - }) @PutMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) @@ -185,7 +184,6 @@ public class TenantController extends BaseController { @ApiOperation(value = "deleteTenantById", notes = "DELETE_TENANT_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "id", value = "TENANT_ID", required = true, dataType = "Int", example = "100") - }) @DeleteMapping(value = "/{id}") @ResponseStatus(HttpStatus.OK) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/EnvironmentDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/EnvironmentDto.java new file mode 100644 index 0000000000..a89d34fe4a --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/EnvironmentDto.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto; + +import java.util.Date; +import java.util.List; + +/** + * EnvironmentDto + */ +public class EnvironmentDto { + + private int id; + + /** + * environment code + */ + private Long code; + + /** + * environment name + */ + private String name; + + /** + * config content + */ + private String config; + + private String description; + + private List workerGroups; + + /** + * operator user id + */ + private Integer operator; + + private Date createTime; + + private Date updateTime; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Long getCode() { + return this.code; + } + + public void setCode(Long code) { + this.code = code; + } + + public String getConfig() { + return this.config; + } + + public void setConfig(String config) { + this.config = config; + } + + public String getDescription() { + return this.description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Integer getOperator() { + return this.operator; + } + + public void setOperator(Integer operator) { + this.operator = operator; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public List getWorkerGroups() { + return workerGroups; + } + + public void setWorkerGroups(List workerGroups) { + this.workerGroups = workerGroups; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 40dc05c593..e2a79e819d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -317,7 +317,22 @@ public enum Status { LIST_PAGING_ALERT_PLUGIN_INSTANCE_ERROR(110011, "query plugin instance page error", "分页查询告警实例失败"), DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED(110012, "failed to delete the alert instance, there is an alarm group associated with this alert instance", "删除告警实例失败，存在与此告警实例关联的警报组"), - PROCESS_DEFINITION_VERSION_IS_USED(110013,"this process definition version is used","此工作流定义版本被使用"); + PROCESS_DEFINITION_VERSION_IS_USED(110013,"this process definition version is used","此工作流定义版本被使用"), + + CREATE_ENVIRONMENT_ERROR(120001, "create environment error", "创建环境失败"), + ENVIRONMENT_NAME_EXISTS(120002,"this enviroment name [{0}] already exists","环境名称[{0}]已经存在"), + ENVIRONMENT_NAME_IS_NULL(120003,"this enviroment name shouldn't be empty.","环境名称不能为空"), + ENVIRONMENT_CONFIG_IS_NULL(120004,"this enviroment config shouldn't be empty.","环境配置信息不能为空"), + UPDATE_ENVIRONMENT_ERROR(120005, "update environment [{0}] info error", "更新环境[{0}]信息失败"), + DELETE_ENVIRONMENT_ERROR(120006, "delete environment error", "删除环境信息失败"), + DELETE_ENVIRONMENT_RELATED_TASK_EXISTS(120007, "this environment has been used in tasks,so you can't delete it.", "该环境已经被任务使用，所以不能删除该环境信息"), + QUERY_ENVIRONMENT_BY_NAME_ERROR(1200008, "not found environment [{0}] ", "查询环境名称[{0}]信息不存在"), + QUERY_ENVIRONMENT_BY_CODE_ERROR(1200009, "not found environment [{0}] ", "查询环境编码[{0}]不存在"), + QUERY_ENVIRONMENT_ERROR(1200010, "login user query environment error", "分页查询环境列表错误"), + VERIFY_ENVIRONMENT_ERROR(1200011, "verify environment error", "验证环境信息错误"), + ENVIRONMENT_WORKER_GROUPS_IS_INVALID(1200012, "environment worker groups is invalid format", "环境关联的工作组参数解析错误"), + UPDATE_ENVIRONMENT_WORKER_GROUP_RELATION_ERROR(1200013,"You can't modify the worker group, because the worker group [{0}] and this environment [{1}] already be used in the task [{2}]", + "您不能修改工作组选项，因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中"); private final int code; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java new file mode 100644 index 0000000000..5702980bf5 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.Map; + +/** + * environment service + */ +public interface EnvironmentService { + + /** + * create environment + * + * @param loginUser login user + * @param name environment name + * @param config environment config + * @param desc environment desc + * @param workerGroups worker groups + */ + Map createEnvironment(User loginUser, String name, String config, String desc, String workerGroups); + + /** + * query environment + * + * @param name environment name + */ + Map queryEnvironmentByName(String name); + + /** + * query environment + * + * @param code environment code + */ + Map queryEnvironmentByCode(Long code); + + + /** + * delete environment + * + * @param loginUser login user + * @param code environment code + */ + Map deleteEnvironmentByCode(User loginUser, Long code); + + /** + * update environment + * + * @param loginUser login user + * @param code environment code + * @param name environment name + * @param config environment config + * @param desc environment desc + * @param workerGroups worker groups + */ + Map updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, String workerGroups); + + /** + * query environment paging + * + * @param pageNo page number + * @param searchVal search value + * @param pageSize page size + * @return environment list page + */ + Result queryEnvironmentListPaging(Integer pageNo, Integer pageSize, String searchVal); + + /** + * query all environment + * + * @return all environment list + */ + Map queryAllEnvironmentList(); + + /** + * verify environment name + * + * @param environmentName environment name + * @return true if the environment name not exists, otherwise return false + */ + Map verifyEnvironment(String environmentName); + +} + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java new file mode 100644 index 0000000000..9db770158d --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import java.util.Map; + +/** + * environment worker group relation service + */ +public interface EnvironmentWorkerGroupRelationService { + + /** + * query environment worker group relation + * + * @param environmentCode environment code + */ + Map queryEnvironmentWorkerGroupRelation(Long environmentCode); + + /** + * query all environment worker group relation + * + * @return all relation list + */ + Map queryAllEnvironmentWorkerGroupRelationList(); +} + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 88acff3546..e86881599d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -49,6 +49,7 @@ public interface ExecutorService { * @param warningGroupId notify group id * @param processInstancePriority process instance priority * @param workerGroup worker group name + * @param environmentCode environment code * @param runMode run mode * @param timeout timeout * @param startParams the global param values which pass to new process instance @@ -60,7 +61,7 @@ public interface ExecutorService { FailureStrategy failureStrategy, String startNodeList, TaskDependType taskDependType, WarningType warningType, int warningGroupId, RunMode runMode, - Priority processInstancePriority, String workerGroup, Integer timeout, + Priority processInstancePriority, String workerGroup, Long environmentCode, Integer timeout, Map startParams, Integer expectedParallelismNumber); /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index a5a70693e0..bad9b3596f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -43,6 +43,7 @@ public interface SchedulerService { * @param failureStrategy failure strategy * @param processInstancePriority process instance priority * @param workerGroup worker group + * @param environmentCode environment code * @return create result code */ Map insertSchedule(User loginUser, @@ -53,7 +54,8 @@ public interface SchedulerService { int warningGroupId, FailureStrategy failureStrategy, Priority processInstancePriority, - String workerGroup); + String workerGroup, + Long environmentCode); /** * updateProcessInstance schedule @@ -66,6 +68,7 @@ public interface SchedulerService { * @param warningGroupId warning group id * @param failureStrategy failure strategy * @param workerGroup worker group + * @param environmentCode environment code * @param processInstancePriority process instance priority * @return update result code */ @@ -77,7 +80,8 @@ public interface SchedulerService { int warningGroupId, FailureStrategy failureStrategy, Priority processInstancePriority, - String workerGroup); + String workerGroup, + Long environmentCode); /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 752c5f7b4d..d18c2605ac 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -310,6 +310,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource * @param type data source type * @param connectionParam connectionParam * @return true if connect successfully, otherwise false + * @return true if connect successfully, otherwise false */ @Override public Result checkConnection(DbType type, ConnectionParam connectionParam) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java new file mode 100644 index 0000000000..f0310a1bf4 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java @@ -0,0 +1,463 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.dto.EnvironmentDto; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.EnvironmentService; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; +import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.Environment; +import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; + +import org.apache.commons.collections4.SetUtils; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.core.type.TypeReference; + +/** + * task definition service impl + */ +@Service +public class EnvironmentServiceImpl extends BaseServiceImpl implements EnvironmentService { + + private static final Logger logger = LoggerFactory.getLogger(EnvironmentServiceImpl.class); + + @Autowired + private EnvironmentMapper environmentMapper; + + @Autowired + private EnvironmentWorkerGroupRelationMapper relationMapper; + + @Autowired + private TaskDefinitionMapper taskDefinitionMapper; + + /** + * create environment + * + * @param loginUser login user + * @param name environment name + * @param config environment config + * @param desc environment desc + * @param workerGroups worker groups + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public Map createEnvironment(User loginUser, String name, String config, String desc, String workerGroups) { + Map result = new HashMap<>(); + if (isNotAdmin(loginUser, result)) { + return result; + } + + Map checkResult = checkParams(name,config,workerGroups); + if (checkResult.get(Constants.STATUS) != Status.SUCCESS) { + return checkResult; + } + + Environment environment = environmentMapper.queryByEnvironmentName(name); + if (environment != null) { + putMsg(result, Status.ENVIRONMENT_NAME_EXISTS, name); + return result; + } + + Environment env = new Environment(); + env.setName(name); + env.setConfig(config); + env.setDescription(desc); + env.setOperator(loginUser.getId()); + env.setCreateTime(new Date()); + env.setUpdateTime(new Date()); + long code = 0L; + try { + code = SnowFlakeUtils.getInstance().nextId(); + env.setCode(code); + } catch (SnowFlakeException e) { + logger.error("Environment code get error, ", e); + } + if (code == 0L) { + putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating environment code"); + return result; + } + + if (environmentMapper.insert(env) > 0) { + if (StringUtils.isNotEmpty(workerGroups)) { + List workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference>(){}); + if (CollectionUtils.isNotEmpty(workerGroupList)) { + workerGroupList.stream().forEach(workerGroup -> { + if (StringUtils.isNotEmpty(workerGroup)) { + EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); + relation.setEnvironmentCode(env.getCode()); + relation.setWorkerGroup(workerGroup); + relation.setOperator(loginUser.getId()); + relation.setCreateTime(new Date()); + relation.setUpdateTime(new Date()); + relationMapper.insert(relation); + } + }); + } + } + result.put(Constants.DATA_LIST, env.getCode()); + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.CREATE_ENVIRONMENT_ERROR); + } + return result; + } + + /** + * query environment paging + * + * @param pageNo page number + * @param searchVal search value + * @param pageSize page size + * @return environment list page + */ + @Override + public Result queryEnvironmentListPaging(Integer pageNo, Integer pageSize, String searchVal) { + Result result = new Result(); + + Page page = new Page<>(pageNo, pageSize); + + IPage environmentIPage = environmentMapper.queryEnvironmentListPaging(page, searchVal); + + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + pageInfo.setTotal((int) environmentIPage.getTotal()); + + if (CollectionUtils.isNotEmpty(environmentIPage.getRecords())) { + Map> relationMap = relationMapper.selectList(null).stream() + .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList()))); + + List dtoList = environmentIPage.getRecords().stream().map(environment -> { + EnvironmentDto dto = new EnvironmentDto(); + BeanUtils.copyProperties(environment,dto); + List workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList()); + dto.setWorkerGroups(workerGroups); + return dto; + }).collect(Collectors.toList()); + + pageInfo.setTotalList(dtoList); + } else { + pageInfo.setTotalList(new ArrayList<>()); + } + + result.setData(pageInfo); + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * query all environment + * + * @return all environment list + */ + @Override + public Map queryAllEnvironmentList() { + Map result = new HashMap<>(); + List environmentList = environmentMapper.queryAllEnvironmentList(); + + if (CollectionUtils.isNotEmpty(environmentList)) { + Map> relationMap = relationMapper.selectList(null).stream() + .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList()))); + + List dtoList = environmentList.stream().map(environment -> { + EnvironmentDto dto = new EnvironmentDto(); + BeanUtils.copyProperties(environment,dto); + List workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList()); + dto.setWorkerGroups(workerGroups); + return dto; + }).collect(Collectors.toList()); + result.put(Constants.DATA_LIST,dtoList); + } else { + result.put(Constants.DATA_LIST, new ArrayList<>()); + } + + putMsg(result,Status.SUCCESS); + return result; + } + + /** + * query environment + * + * @param code environment code + */ + @Override + public Map queryEnvironmentByCode(Long code) { + Map result = new HashMap<>(); + + Environment env = environmentMapper.queryByEnvironmentCode(code); + + if (env == null) { + putMsg(result, Status.QUERY_ENVIRONMENT_BY_CODE_ERROR, code); + } else { + List workerGroups = relationMapper.queryByEnvironmentCode(env.getCode()).stream() + .map(item -> item.getWorkerGroup()) + .collect(Collectors.toList()); + + EnvironmentDto dto = new EnvironmentDto(); + BeanUtils.copyProperties(env,dto); + dto.setWorkerGroups(workerGroups); + result.put(Constants.DATA_LIST, dto); + putMsg(result, Status.SUCCESS); + } + return result; + } + + /** + * query environment + * + * @param name environment name + */ + @Override + public Map queryEnvironmentByName(String name) { + Map result = new HashMap<>(); + + Environment env = environmentMapper.queryByEnvironmentName(name); + if (env == null) { + putMsg(result, Status.QUERY_ENVIRONMENT_BY_NAME_ERROR, name); + } else { + List workerGroups = relationMapper.queryByEnvironmentCode(env.getCode()).stream() + .map(item -> item.getWorkerGroup()) + .collect(Collectors.toList()); + + EnvironmentDto dto = new EnvironmentDto(); + BeanUtils.copyProperties(env,dto); + dto.setWorkerGroups(workerGroups); + result.put(Constants.DATA_LIST, dto); + putMsg(result, Status.SUCCESS); + } + return result; + } + + /** + * delete environment + * + * @param loginUser login user + * @param code environment code + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public Map deleteEnvironmentByCode(User loginUser, Long code) { + Map result = new HashMap<>(); + if (isNotAdmin(loginUser, result)) { + return result; + } + + Integer relatedTaskNumber = taskDefinitionMapper + .selectCount(new QueryWrapper().lambda().eq(TaskDefinition::getEnvironmentCode,code)); + + if (relatedTaskNumber > 0) { + putMsg(result, Status.DELETE_ENVIRONMENT_RELATED_TASK_EXISTS); + return result; + } + + int delete = environmentMapper.deleteByCode(code); + if (delete > 0) { + relationMapper.delete(new QueryWrapper() + .lambda() + .eq(EnvironmentWorkerGroupRelation::getEnvironmentCode,code)); + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.DELETE_ENVIRONMENT_ERROR); + } + return result; + } + + /** + * update environment + * + * @param loginUser login user + * @param code environment code + * @param name environment name + * @param config environment config + * @param desc environment desc + * @param workerGroups worker groups + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public Map updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, String workerGroups) { + Map result = new HashMap<>(); + if (isNotAdmin(loginUser, result)) { + return result; + } + + Map checkResult = checkParams(name,config,workerGroups); + if (checkResult.get(Constants.STATUS) != Status.SUCCESS) { + return checkResult; + } + + Environment environment = environmentMapper.queryByEnvironmentName(name); + if (environment != null && !environment.getCode().equals(code)) { + putMsg(result, Status.ENVIRONMENT_NAME_EXISTS, name); + return result; + } + + Set workerGroupSet; + if (StringUtils.isNotEmpty(workerGroups)) { + workerGroupSet = JSONUtils.parseObject(workerGroups, new TypeReference>() {}); + } else { + workerGroupSet = new TreeSet<>(); + } + + Set existWorkerGroupSet = relationMapper + .queryByEnvironmentCode(code) + .stream() + .map(item -> item.getWorkerGroup()) + .collect(Collectors.toSet()); + + Set deleteWorkerGroupSet = SetUtils.difference(existWorkerGroupSet,workerGroupSet).toSet(); + Set addWorkerGroupSet = SetUtils.difference(workerGroupSet,existWorkerGroupSet).toSet(); + + // verify whether the relation of this environment and worker groups can be adjusted + checkResult = checkUsedEnvironmentWorkerGroupRelation(deleteWorkerGroupSet, name, code); + if (checkResult.get(Constants.STATUS) != Status.SUCCESS) { + return checkResult; + } + + Environment env = new Environment(); + env.setCode(code); + env.setName(name); + env.setConfig(config); + env.setDescription(desc); + env.setOperator(loginUser.getId()); + env.setUpdateTime(new Date()); + + int update = environmentMapper.update(env, new UpdateWrapper().lambda().eq(Environment::getCode,code)); + if (update > 0) { + deleteWorkerGroupSet.stream().forEach(key -> { + if (StringUtils.isNotEmpty(key)) { + relationMapper.delete(new QueryWrapper() + .lambda() + .eq(EnvironmentWorkerGroupRelation::getEnvironmentCode,code)); + } + }); + addWorkerGroupSet.stream().forEach(key -> { + if (StringUtils.isNotEmpty(key)) { + EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); + relation.setEnvironmentCode(code); + relation.setWorkerGroup(key); + relation.setUpdateTime(new Date()); + relation.setCreateTime(new Date()); + relation.setOperator(loginUser.getId()); + relationMapper.insert(relation); + } + }); + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.UPDATE_ENVIRONMENT_ERROR, name); + } + return result; + } + + + + /** + * verify environment name + * + * @param environmentName environment name + * @return true if the environment name not exists, otherwise return false + */ + @Override + public Map verifyEnvironment(String environmentName) { + Map result = new HashMap<>(); + + if (StringUtils.isEmpty(environmentName)) { + putMsg(result, Status.ENVIRONMENT_NAME_IS_NULL); + return result; + } + + Environment environment = environmentMapper.queryByEnvironmentName(environmentName); + if (environment != null) { + putMsg(result, Status.ENVIRONMENT_NAME_EXISTS, environmentName); + return result; + } + + result.put(Constants.STATUS, Status.SUCCESS); + return result; + } + + private Map checkUsedEnvironmentWorkerGroupRelation(Set deleteKeySet,String environmentName, Long environmentCode) { + Map result = new HashMap<>(); + for (String workerGroup : deleteKeySet) { + TaskDefinition taskDefinition = taskDefinitionMapper + .selectOne(new QueryWrapper().lambda() + .eq(TaskDefinition::getEnvironmentCode,environmentCode) + .eq(TaskDefinition::getWorkerGroup,workerGroup)); + + if (Objects.nonNull(taskDefinition)) { + putMsg(result, Status.UPDATE_ENVIRONMENT_WORKER_GROUP_RELATION_ERROR,workerGroup,environmentName,taskDefinition.getName()); + return result; + } + } + result.put(Constants.STATUS, Status.SUCCESS); + return result; + } + + public Map checkParams(String name, String config, String workerGroups) { + Map result = new HashMap<>(); + if (StringUtils.isEmpty(name)) { + putMsg(result, Status.ENVIRONMENT_NAME_IS_NULL); + return result; + } + if (StringUtils.isEmpty(config)) { + putMsg(result, Status.ENVIRONMENT_CONFIG_IS_NULL); + return result; + } + if (StringUtils.isNotEmpty(workerGroups)) { + List workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference>(){}); + if (Objects.isNull(workerGroupList)) { + putMsg(result, Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID); + return result; + } + } + result.put(Constants.STATUS, Status.SUCCESS); + return result; + } + +} + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java new file mode 100644 index 0000000000..7fa7104ebf --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.EnvironmentWorkerGroupRelationService; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * task definition service impl + */ +@Service +public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl implements + EnvironmentWorkerGroupRelationService { + + private static final Logger logger = LoggerFactory.getLogger(EnvironmentWorkerGroupRelationServiceImpl.class); + + @Autowired + private EnvironmentWorkerGroupRelationMapper environmentWorkerGroupRelationMapper; + + /** + * query environment worker group relation + * + * @param environmentCode environment code + */ + @Override + public Map queryEnvironmentWorkerGroupRelation(Long environmentCode) { + Map result = new HashMap<>(); + List relations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(environmentCode); + result.put(Constants.DATA_LIST, relations); + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * query all environment worker group relation + * + * @return all relation list + */ + @Override + public Map queryAllEnvironmentWorkerGroupRelationList() { + Map result = new HashMap<>(); + + List relations = environmentWorkerGroupRelationMapper.selectList(null); + + result.put(Constants.DATA_LIST,relations); + putMsg(result,Status.SUCCESS); + return result; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 35f9330139..465ef7e95d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -53,6 +53,8 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand; +import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; @@ -98,6 +100,9 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ @Autowired private ProcessService processService; + @Autowired + StateEventCallbackService stateEventCallbackService; + /** * execute process instance * @@ -113,6 +118,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ * @param warningGroupId notify group id * @param processInstancePriority process instance priority * @param workerGroup worker group name + * @param environmentCode environment code * @param runMode run mode * @param timeout timeout * @param startParams the global param values which pass to new process instance @@ -125,7 +131,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ FailureStrategy failureStrategy, String startNodeList, TaskDependType taskDependType, WarningType warningType, int warningGroupId, RunMode runMode, - Priority processInstancePriority, String workerGroup, Integer timeout, + Priority processInstancePriority, String workerGroup, Long environmentCode,Integer timeout, Map startParams, Integer expectedParallelismNumber) { Project project = projectMapper.queryByCode(projectCode); //check user access for project @@ -163,7 +169,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ */ int create = this.createCommand(commandType, processDefinition.getCode(), taskDependType, failureStrategy, startNodeList, cronTime, warningType, loginUser.getId(), - warningGroupId, runMode, processInstancePriority, workerGroup, startParams, expectedParallelismNumber); + warningGroupId, runMode, processInstancePriority, workerGroup, environmentCode, startParams, expectedParallelismNumber); if (create > 0) { processDefinition.setWarningGroupId(warningGroupId); @@ -381,6 +387,13 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ // determine whether the process is normal if (update > 0) { + String host = processInstance.getHost(); + String address = host.split(":")[0]; + int port = Integer.parseInt(host.split(":")[1]); + StateEventChangeCommand stateEventChangeCommand = new StateEventChangeCommand( + processInstance.getId(), 0, processInstance.getState(), processInstance.getId(), 0 + ); + stateEventCallbackService.sendResult(address, port, stateEventChangeCommand.convert2Command()); putMsg(result, Status.SUCCESS); } else { putMsg(result, Status.EXECUTE_PROCESS_INSTANCE_ERROR); @@ -485,13 +498,14 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ * @param runMode runMode * @param processInstancePriority processInstancePriority * @param workerGroup workerGroup + * @param environmentCode environmentCode * @return command id */ private int createCommand(CommandType commandType, long processDefineCode, TaskDependType nodeDep, FailureStrategy failureStrategy, String startNodeList, String schedule, WarningType warningType, int executorId, int warningGroupId, - RunMode runMode, Priority processInstancePriority, String workerGroup, + RunMode runMode, Priority processInstancePriority, String workerGroup, Long environmentCode, Map startParams, Integer expectedParallelismNumber) { /** @@ -527,6 +541,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ command.setWarningGroupId(warningGroupId); command.setProcessInstancePriority(processInstancePriority); command.setWorkerGroup(workerGroup); + command.setEnvironmentCode(environmentCode); Date start = null; Date end = null; @@ -537,7 +552,6 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ end = DateUtils.getScheduleDate(interval[1]); } } - // determine whether to complement if (commandType == CommandType.COMPLEMENT_DATA) { runMode = (runMode == null) ? RunMode.RUN_MODE_SERIAL : runMode; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 68007ccba1..a40f821ce3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -46,6 +46,7 @@ import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.DagData; +import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 6c23e3987e..46b8ecd10c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -576,7 +576,12 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce return result; } - processService.removeTaskLogFile(processInstanceId); + try { + processService.removeTaskLogFile(processInstanceId); + } catch (Exception e) { + logger.error("remove task log failed", e); + } + // delete database cascade int delete = processService.deleteWorkProcessInstanceById(processInstanceId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 80603ef3a8..4a8b8e8cc3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -187,6 +187,19 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic return checkResult; } + @Override + public boolean hasProjectAndPerm(User loginUser, Project project, Result result) { + boolean checkResult = false; + if (project == null) { + putMsg(result, Status.PROJECT_NOT_FOUNT, ""); + } else if (!checkReadPermission(loginUser, project)) { + putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName()); + } else { + checkResult = true; + } + return checkResult; + } + /** * admin can view all projects * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java index d88a820768..d03de22a91 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java @@ -106,6 +106,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe * @param failureStrategy failure strategy * @param processInstancePriority process instance priority * @param workerGroup worker group + * @param environmentCode environment code * @return create result code */ @Override @@ -118,7 +119,8 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe int warningGroupId, FailureStrategy failureStrategy, Priority processInstancePriority, - String workerGroup) { + String workerGroup, + Long environmentCode) { Map result = new HashMap<>(); @@ -170,6 +172,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe scheduleObj.setReleaseState(ReleaseState.OFFLINE); scheduleObj.setProcessInstancePriority(processInstancePriority); scheduleObj.setWorkerGroup(workerGroup); + scheduleObj.setEnvironmentCode(environmentCode); scheduleMapper.insert(scheduleObj); /** @@ -197,6 +200,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe * @param warningGroupId warning group id * @param failureStrategy failure strategy * @param workerGroup worker group + * @param environmentCode environment code * @param processInstancePriority process instance priority * @return update result code */ @@ -210,7 +214,8 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe int warningGroupId, FailureStrategy failureStrategy, Priority processInstancePriority, - String workerGroup) { + String workerGroup, + Long environmentCode) { Map result = new HashMap<>(); Project project = projectMapper.queryByCode(projectCode); @@ -273,6 +278,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe } schedule.setWorkerGroup(workerGroup); + schedule.setEnvironmentCode(environmentCode); schedule.setUpdateTime(now); schedule.setProcessInstancePriority(processInstancePriority); scheduleMapper.updateById(schedule); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index b198013de1..9f4b8cb8e5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -110,7 +110,6 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst putMsg(result,status); return result; } - int[] statusArray = null; if (stateType != null) { statusArray = new int[]{stateType.ordinal()}; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java index a0f3ed4c9d..0510bcd0df 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java @@ -49,6 +49,7 @@ public class AlertGroupControllerTest extends AbstractControllerTest { paramsMap.add("groupName","cxc test group name"); paramsMap.add("groupType", AlertType.EMAIL.toString()); paramsMap.add("description","cxc junit 测试告警描述"); + paramsMap.add("alertInstanceIds", ""); MvcResult mvcResult = mockMvc.perform(post("/alert-group/create") .header("sessionId", sessionId) .params(paramsMap)) @@ -93,13 +94,10 @@ public class AlertGroupControllerTest extends AbstractControllerTest { } @Test - public void testUpdateAlertgroup() throws Exception { + public void testQueryAlertGroupById() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("id","22"); - paramsMap.add("groupName", "hd test group name"); - paramsMap.add("groupType",AlertType.EMAIL.toString()); - paramsMap.add("description","update alter group"); - MvcResult mvcResult = mockMvc.perform(post("/alert-group/update") + MvcResult mvcResult = mockMvc.perform(post("/alert-group/query") .header("sessionId", sessionId) .params(paramsMap)) .andExpect(status().isOk()) @@ -111,10 +109,14 @@ public class AlertGroupControllerTest extends AbstractControllerTest { } @Test - public void testVerifyGroupName() throws Exception { + public void testUpdateAlertgroup() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("groupName","hd test group name"); - MvcResult mvcResult = mockMvc.perform(get("/alert-group/verify-group-name") + paramsMap.add("id","22"); + paramsMap.add("groupName", "cxc test group name"); + paramsMap.add("groupType",AlertType.EMAIL.toString()); + paramsMap.add("description","update alter group"); + paramsMap.add("alertInstanceIds", ""); + MvcResult mvcResult = mockMvc.perform(post("/alert-group/update") .header("sessionId", sessionId) .params(paramsMap)) .andExpect(status().isOk()) @@ -126,7 +128,7 @@ public class AlertGroupControllerTest extends AbstractControllerTest { } @Test - public void testVerifyGroupNameNotExit() throws Exception { + public void testVerifyGroupName() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("groupName","cxc test group name"); MvcResult mvcResult = mockMvc.perform(get("/alert-group/verify-group-name") @@ -141,12 +143,10 @@ public class AlertGroupControllerTest extends AbstractControllerTest { } @Test - public void testGrantUser() throws Exception { + public void testVerifyGroupNameNotExit() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("alertgroupId","2"); - paramsMap.add("userIds","2"); - - MvcResult mvcResult = mockMvc.perform(post("/alert-group/grant-user") + paramsMap.add("groupName","cxc test group name"); + MvcResult mvcResult = mockMvc.perform(get("/alert-group/verify-group-name") .header("sessionId", sessionId) .params(paramsMap)) .andExpect(status().isOk()) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java index d63ffff30e..12467f437f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataAnalysisControllerTest.java @@ -21,7 +21,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.Project; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java index e7863739de..b0bb516f66 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java @@ -22,7 +22,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; import org.apache.dolphinscheduler.common.utils.JSONUtils; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/EnvironmentControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/EnvironmentControllerTest.java new file mode 100644 index 0000000000..7ba51ae785 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/EnvironmentControllerTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.controller; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.Preconditions; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import com.fasterxml.jackson.core.type.TypeReference; + +/** + * environment controller test + */ +public class EnvironmentControllerTest extends AbstractControllerTest { + + private static Logger logger = LoggerFactory.getLogger(EnvironmentControllerTest.class); + + private String environmentCode; + + public static final String environmentName = "Env1"; + + public static final String config = "this is config content"; + + public static final String desc = "this is environment description"; + + @Before + public void before() throws Exception { + testCreateEnvironment(); + } + + @After + public void after() throws Exception { + testDeleteEnvironment(); + } + + public void testCreateEnvironment() throws Exception { + + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("name",environmentName); + paramsMap.add("config",config); + paramsMap.add("description",desc); + + MvcResult mvcResult = mockMvc.perform(post("/environment/create") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() {}); + logger.info(result.toString()); + Assert.assertTrue(result != null && result.isSuccess()); + Assert.assertNotNull(result.getData()); + logger.info("create environment return result:{}", mvcResult.getResponse().getContentAsString()); + + environmentCode = (String)result.getData(); + } + + @Test + public void testUpdateEnvironment() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("code", environmentCode); + paramsMap.add("name","environment_test_update"); + paramsMap.add("config","this is config content"); + paramsMap.add("desc","the test environment update"); + + MvcResult mvcResult = mockMvc.perform(post("/environment/update") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + logger.info(result.toString()); + Assert.assertTrue(result != null && result.isSuccess()); + logger.info("update environment return result:{}", mvcResult.getResponse().getContentAsString()); + + } + + @Test + public void testQueryEnvironmentByCode() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("environmentCode", environmentCode); + + MvcResult mvcResult = mockMvc.perform(get("/environment/query-by-code") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + logger.info(result.toString()); + Assert.assertTrue(result != null && result.isSuccess()); + logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("query environment by id :{}, return result:{}", environmentCode, mvcResult.getResponse().getContentAsString()); + + } + + @Test + public void testQueryEnvironmentListPaging() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("searchVal","test"); + paramsMap.add("pageSize","2"); + paramsMap.add("pageNo","2"); + + MvcResult mvcResult = mockMvc.perform(get("/environment/list-paging") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + logger.info(result.toString()); + Assert.assertTrue(result != null && result.isSuccess()); + logger.info("query list-paging environment return result:{}", mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testQueryAllEnvironmentList() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + + MvcResult mvcResult = mockMvc.perform(get("/environment/query-environment-list") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + logger.info(result.toString()); + Assert.assertTrue(result != null && result.isSuccess()); + logger.info("query all environment return result:{}", mvcResult.getResponse().getContentAsString()); + + } + + @Test + public void testVerifyEnvironment() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("environmentName",environmentName); + + MvcResult mvcResult = mockMvc.perform(post("/environment/verify-environment") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + logger.info(result.toString()); + Assert.assertTrue(result.isStatus(Status.ENVIRONMENT_NAME_EXISTS)); + logger.info("verify environment return result:{}", mvcResult.getResponse().getContentAsString()); + + } + + private void testDeleteEnvironment() throws Exception { + Preconditions.checkNotNull(environmentCode); + + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("environmentCode", environmentCode); + + MvcResult mvcResult = mockMvc.perform(post("/environment/delete") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + logger.info(result.toString()); + Assert.assertTrue(result != null && result.isSuccess()); + logger.info("delete environment return result:{}", mvcResult.getResponse().getContentAsString()); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java index 4dafaaa61f..f271a849b4 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoggerControllerTest.java @@ -21,7 +21,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index 5c56230832..1a852465b5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -21,7 +21,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java index 057a73adfe..60beef1a0e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java @@ -21,7 +21,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java index fb4b0cee4a..2ab09325ed 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java @@ -22,7 +22,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java new file mode 100644 index 0000000000..b9b95ecae8 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.EnvironmentServiceImpl; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.Environment; +import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.assertj.core.util.Lists; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +/** + * environment service test + */ +@RunWith(MockitoJUnitRunner.class) +public class EnvironmentServiceTest { + + public static final Logger logger = LoggerFactory.getLogger(EnvironmentServiceTest.class); + + @InjectMocks + private EnvironmentServiceImpl environmentService; + + @Mock + private EnvironmentMapper environmentMapper; + + @Mock + private EnvironmentWorkerGroupRelationMapper relationMapper; + + @Mock + private TaskDefinitionMapper taskDefinitionMapper; + + public static final String testUserName = "environmentServerTest"; + + public static final String environmentName = "Env1"; + + public static final String workerGroups = "[\"default\"]"; + + @Before + public void setUp(){ + } + + @After + public void after(){ + } + + @Test + public void testCreateEnvironment() { + User loginUser = getGeneralUser(); + Map result = environmentService.createEnvironment(loginUser,environmentName,getConfig(),getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + loginUser = getAdminUser(); + result = environmentService.createEnvironment(loginUser,environmentName,"",getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_CONFIG_IS_NULL, result.get(Constants.STATUS)); + + result = environmentService.createEnvironment(loginUser,"",getConfig(),getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_NAME_IS_NULL, result.get(Constants.STATUS)); + + result = environmentService.createEnvironment(loginUser,environmentName,getConfig(),getDesc(),"test"); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID, result.get(Constants.STATUS)); + + Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment()); + result = environmentService.createEnvironment(loginUser,environmentName,getConfig(),getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS)); + + Mockito.when(environmentMapper.insert(Mockito.any(Environment.class))).thenReturn(1); + Mockito.when(relationMapper.insert(Mockito.any(EnvironmentWorkerGroupRelation.class))).thenReturn(1); + result = environmentService.createEnvironment(loginUser,"testName","test","test",workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + } + + @Test + public void testCheckParams() { + Map result = environmentService.checkParams(environmentName,getConfig(),"test"); + Assert.assertEquals(Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID, result.get(Constants.STATUS)); + } + + @Test + public void testUpdateEnvironmentByCode() { + User loginUser = getGeneralUser(); + Map result = environmentService.updateEnvironmentByCode(loginUser,1L,environmentName,getConfig(),getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + loginUser = getAdminUser(); + result = environmentService.updateEnvironmentByCode(loginUser,1L,environmentName,"",getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_CONFIG_IS_NULL, result.get(Constants.STATUS)); + + result = environmentService.updateEnvironmentByCode(loginUser,1L,"",getConfig(),getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_NAME_IS_NULL, result.get(Constants.STATUS)); + + result = environmentService.updateEnvironmentByCode(loginUser,1L,environmentName,getConfig(),getDesc(),"test"); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID, result.get(Constants.STATUS)); + + Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment()); + result = environmentService.updateEnvironmentByCode(loginUser,2L,environmentName,getConfig(),getDesc(),workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS)); + + Mockito.when(environmentMapper.update(Mockito.any(Environment.class),Mockito.any(Wrapper.class))).thenReturn(1); + result = environmentService.updateEnvironmentByCode(loginUser,1L,"testName","test","test",workerGroups); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + } + + @Test + public void testQueryAllEnvironmentList() { + Mockito.when(environmentMapper.queryAllEnvironmentList()).thenReturn(Lists.newArrayList(getEnvironment())); + Map result = environmentService.queryAllEnvironmentList(); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + List list = (List)(result.get(Constants.DATA_LIST)); + Assert.assertEquals(1,list.size()); + } + + @Test + public void testQueryEnvironmentListPaging() { + IPage page = new Page<>(1, 10); + page.setRecords(getList()); + page.setTotal(1L); + Mockito.when(environmentMapper.queryEnvironmentListPaging(Mockito.any(Page.class), Mockito.eq(environmentName))).thenReturn(page); + + Result result = environmentService.queryEnvironmentListPaging(1, 10, environmentName); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.getData(); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList())); + } + + @Test + public void testQueryEnvironmentByName() { + Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(null); + Map result = environmentService.queryEnvironmentByName(environmentName); + logger.info(result.toString()); + Assert.assertEquals(Status.QUERY_ENVIRONMENT_BY_NAME_ERROR,result.get(Constants.STATUS)); + + Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment()); + result = environmentService.queryEnvironmentByName(environmentName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testQueryEnvironmentByCode() { + Mockito.when(environmentMapper.queryByEnvironmentCode(1L)).thenReturn(null); + Map result = environmentService.queryEnvironmentByCode(1L); + logger.info(result.toString()); + Assert.assertEquals(Status.QUERY_ENVIRONMENT_BY_CODE_ERROR,result.get(Constants.STATUS)); + + Mockito.when(environmentMapper.queryByEnvironmentCode(1L)).thenReturn(getEnvironment()); + result = environmentService.queryEnvironmentByCode(1L); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testDeleteEnvironmentByCode() { + User loginUser = getGeneralUser(); + Map result = environmentService.deleteEnvironmentByCode(loginUser,1L); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + loginUser = getAdminUser(); + Mockito.when(taskDefinitionMapper.selectCount(Mockito.any(LambdaQueryWrapper.class))).thenReturn(1); + result = environmentService.deleteEnvironmentByCode(loginUser,1L); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_ENVIRONMENT_RELATED_TASK_EXISTS, result.get(Constants.STATUS)); + + Mockito.when(taskDefinitionMapper.selectCount(Mockito.any(LambdaQueryWrapper.class))).thenReturn(0); + Mockito.when(environmentMapper.deleteByCode(1L)).thenReturn(1); + result = environmentService.deleteEnvironmentByCode(loginUser,1L); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + @Test + public void testVerifyEnvironment() { + Map result = environmentService.verifyEnvironment(""); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_NAME_IS_NULL, result.get(Constants.STATUS)); + + Mockito.when(environmentMapper.queryByEnvironmentName(environmentName)).thenReturn(getEnvironment()); + result = environmentService.verifyEnvironment(environmentName); + logger.info(result.toString()); + Assert.assertEquals(Status.ENVIRONMENT_NAME_EXISTS, result.get(Constants.STATUS)); + } + + private Environment getEnvironment() { + Environment environment = new Environment(); + environment.setId(1); + environment.setCode(1L); + environment.setName(environmentName); + environment.setConfig(getConfig()); + environment.setDescription(getDesc()); + environment.setOperator(1); + return environment; + } + + /** + * create an environment description + */ + private String getDesc() { + return "create an environment to test "; + } + + /** + * create an environment config + */ + private String getConfig() { + return "export HADOOP_HOME=/opt/hadoop-2.6.5\n" + + "export HADOOP_CONF_DIR=/etc/hadoop/conf\n" + + "export SPARK_HOME1=/opt/soft/spark1\n" + + "export SPARK_HOME2=/opt/soft/spark2\n" + + "export PYTHON_HOME=/opt/soft/python\n" + + "export JAVA_HOME=/opt/java/jdk1.8.0_181-amd64\n" + + "export HIVE_HOME=/opt/soft/hive\n" + + "export FLINK_HOME=/opt/soft/flink\n" + + "export DATAX_HOME=/opt/soft/datax\n" + + "export YARN_CONF_DIR=\"/etc/hadoop/conf\"\n" + + "\n" + + "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH\n" + + "\n" + + "export HADOOP_CLASSPATH=`hadoop classpath`\n" + + "\n" + + "#echo \"HADOOP_CLASSPATH=\"$HADOOP_CLASSPATH"; + } + + /** + * create general user + */ + private User getGeneralUser() { + User loginUser = new User(); + loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setUserName(testUserName); + loginUser.setId(1); + return loginUser; + } + + /** + * create admin user + */ + private User getAdminUser() { + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setUserName(testUserName); + loginUser.setId(1); + return loginUser; + } + + private List getList() { + List list = new ArrayList<>(); + list.add(getEnvironment()); + return list; + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationServiceTest.java new file mode 100644 index 0000000000..5a3026fd1f --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationServiceTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.EnvironmentWorkerGroupRelationServiceImpl; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper; + +import java.util.Map; + +import org.assertj.core.util.Lists; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * environment service test + */ +@RunWith(MockitoJUnitRunner.class) +public class EnvironmentWorkerGroupRelationServiceTest { + + public static final Logger logger = LoggerFactory.getLogger(EnvironmentWorkerGroupRelationServiceTest.class); + + @InjectMocks + private EnvironmentWorkerGroupRelationServiceImpl relationService; + + @Mock + private EnvironmentWorkerGroupRelationMapper relationMapper; + + @Test + public void testQueryEnvironmentWorkerGroupRelation() { + Mockito.when(relationMapper.queryByEnvironmentCode(1L)).thenReturn(Lists.newArrayList(new EnvironmentWorkerGroupRelation())); + Map result = relationService.queryEnvironmentWorkerGroupRelation(1L); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testQueryAllEnvironmentWorkerGroupRelationList() { + Mockito.when(relationMapper.selectList(Mockito.any())).thenReturn(Lists.newArrayList(new EnvironmentWorkerGroupRelation())); + Map result = relationService.queryAllEnvironmentWorkerGroupRelationList(); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 9abf6c483b..373261c2e1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -82,25 +83,36 @@ public class ProcessDefinitionServiceTest { @InjectMocks private ProcessDefinitionServiceImpl processDefinitionService; + @Mock private ProcessDefinitionMapper processDefineMapper; + @Mock private ProcessTaskRelationMapper processTaskRelationMapper; + @Mock private ProjectMapper projectMapper; + @Mock private ProjectServiceImpl projectService; + @Mock private ScheduleMapper scheduleMapper; + @Mock private ProcessService processService; + @Mock private ProcessInstanceService processInstanceService; + @Mock private TaskInstanceMapper taskInstanceMapper; @Mock private TenantMapper tenantMapper; + @Mock + private DataSourceMapper dataSourceMapper; + @Test public void testQueryProcessDefinitionList() { long projectCode = 1L; @@ -466,7 +478,7 @@ public class ProcessDefinitionServiceTest { Assert.assertEquals(Status.SUCCESS, processNotExistRes.get(Constants.STATUS)); //process exist - Mockito.when(processDefineMapper.verifyByDefineName(project.getCode(), "test_pdf")).thenReturn(getProcessDefinition()); + Mockito.when(processDefinitionMapper.verifyByDefineName(project.getCode(), "test_pdf")).thenReturn(getProcessDefinition()); Map processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser, projectCode, "test_pdf"); Assert.assertEquals(Status.PROCESS_DEFINITION_NAME_EXIST, processExistRes.get(Constants.STATUS)); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index fa68525ac1..630769344b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -186,13 +186,13 @@ public class ProcessInstanceServiceTest { "192.168.xx.xx", 1, 10); Assert.assertEquals(Status.SUCCESS.getCode(), (int)successRes.getCode()); - //executor null when(usersService.queryUser(loginUser.getId())).thenReturn(null); when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(-1); Result executorExistRes = processInstanceService.queryProcessInstanceList(loginUser, projectCode, 1, "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", "admin", ExecutionStatus.SUBMITTED_SUCCESS, "192.168.xx.xx", 1, 10); + Assert.assertEquals(Status.SUCCESS.getCode(), (int)executorExistRes.getCode()); //executor name empty diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index 2e9262dd6b..fe8258c7d1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -143,6 +143,11 @@ public class TaskNode { */ private String workerGroup; + /** + * environment code + */ + private Long environmentCode; + /** * task time out */ @@ -262,6 +267,7 @@ public class TaskNode { && Objects.equals(runFlag, taskNode.runFlag) && Objects.equals(dependence, taskNode.dependence) && Objects.equals(workerGroup, taskNode.workerGroup) + && Objects.equals(environmentCode, taskNode.environmentCode) && Objects.equals(conditionResult, taskNode.conditionResult) && CollectionUtils.equalLists(depList, taskNode.depList); } @@ -422,11 +428,20 @@ public class TaskNode { + ", conditionResult='" + conditionResult + '\'' + ", taskInstancePriority=" + taskInstancePriority + ", workerGroup='" + workerGroup + '\'' + + ", environmentCode=" + environmentCode + ", timeout='" + timeout + '\'' + ", delayTime=" + delayTime + '}'; } + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + + public Long getEnvironmentCode() { + return this.environmentCode; + } + public String getSwitchResult() { return switchResult; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java index bef466e7e8..e3088d022b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java @@ -120,6 +120,12 @@ public class Command { @TableField("worker_group") private String workerGroup; + /** + * environment code + */ + @TableField("environment_code") + private Long environmentCode; + public Command() { this.taskDependType = TaskDependType.TASK_POST; this.failureStrategy = FailureStrategy.CONTINUE; @@ -138,6 +144,7 @@ public class Command { int warningGroupId, Date scheduleTime, String workerGroup, + Long environmentCode, Priority processInstancePriority) { this.commandType = commandType; this.executorId = executorId; @@ -151,6 +158,7 @@ public class Command { this.startTime = new Date(); this.updateTime = new Date(); this.workerGroup = workerGroup; + this.environmentCode = environmentCode; this.processInstancePriority = processInstancePriority; } @@ -266,6 +274,14 @@ public class Command { this.workerGroup = workerGroup; } + public Long getEnvironmentCode() { + return this.environmentCode; + } + + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -289,6 +305,11 @@ public class Command { if (workerGroup != null ? workerGroup.equals(command.workerGroup) : command.workerGroup == null) { return false; } + + if (environmentCode != null ? environmentCode.equals(command.environmentCode) : command.environmentCode == null) { + return false; + } + if (commandType != command.commandType) { return false; } @@ -336,6 +357,7 @@ public class Command { result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0); result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0); result = 31 * result + (workerGroup != null ? workerGroup.hashCode() : 0); + result = 31 * result + (environmentCode != null ? environmentCode.hashCode() : 0); return result; } @@ -356,6 +378,7 @@ public class Command { + ", processInstancePriority=" + processInstancePriority + ", updateTime=" + updateTime + ", workerGroup='" + workerGroup + '\'' + + ", environmentCode='" + environmentCode + '\'' + '}'; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Environment.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Environment.java new file mode 100644 index 0000000000..ad0f7148a4 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Environment.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; + +/** + * Environment + */ +@TableName("t_ds_environment") +public class Environment { + + @TableId(value = "id", type = IdType.AUTO) + private int id; + + /** + * environment code + */ + private Long code; + + /** + * environment name + */ + private String name; + + /** + * config content + */ + private String config; + + private String description; + + /** + * operator user id + */ + private Integer operator; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date createTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date updateTime; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Long getCode() { + return this.code; + } + + public void setCode(Long code) { + this.code = code; + } + + public String getConfig() { + return this.config; + } + + public void setConfig(String config) { + this.config = config; + } + + public String getDescription() { + return this.description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Integer getOperator() { + return this.operator; + } + + public void setOperator(Integer operator) { + this.operator = operator; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "Environment{" + + "id= " + id + + ", code= " + code + + ", name= " + name + + ", config= " + config + + ", description= " + description + + ", operator= " + operator + + ", createTime= " + createTime + + ", updateTime= " + updateTime + + "}"; + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/EnvironmentWorkerGroupRelation.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/EnvironmentWorkerGroupRelation.java new file mode 100644 index 0000000000..d1ac972032 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/EnvironmentWorkerGroupRelation.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; + +/** + * EnvironmentWorkerGroupRelation + */ +@TableName("t_ds_environment_worker_group_relation") +public class EnvironmentWorkerGroupRelation { + + @TableId(value = "id", type = IdType.AUTO) + private int id; + + /** + * environment code + */ + private Long environmentCode; + + /** + * worker group id + */ + private String workerGroup; + + /** + * operator user id + */ + private Integer operator; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date createTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date updateTime; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getWorkerGroup() { + return workerGroup; + } + + public void setWorkerGroup(String workerGroup) { + this.workerGroup = workerGroup; + } + + public Long getEnvironmentCode() { + return this.environmentCode; + } + + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + + public Integer getOperator() { + return this.operator; + } + + public void setOperator(Integer operator) { + this.operator = operator; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "EnvironmentWorkerGroupRelation{" + + "id= " + id + + ", environmentCode= " + environmentCode + + ", workerGroup= " + workerGroup + + ", operator= " + operator + + ", createTime= " + createTime + + ", updateTime= " + updateTime + + "}"; + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index c4a5e6070a..d03570d9e8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -115,6 +115,11 @@ public class ErrorCommand { */ private String workerGroup; + /** + * environment code + */ + private Long environmentCode; + public ErrorCommand(){} public ErrorCommand(Command command, String message) { @@ -130,6 +135,7 @@ public class ErrorCommand { this.failureStrategy = command.getFailureStrategy(); this.startTime = command.getStartTime(); this.updateTime = command.getUpdateTime(); + this.environmentCode = command.getEnvironmentCode(); this.processInstancePriority = command.getProcessInstancePriority(); this.message = message; } @@ -254,6 +260,14 @@ public class ErrorCommand { this.message = message; } + public Long getEnvironmentCode() { + return this.environmentCode; + } + + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + @Override public String toString() { return "ErrorCommand{" @@ -272,6 +286,7 @@ public class ErrorCommand { + ", updateTime=" + updateTime + ", message='" + message + '\'' + ", workerGroup='" + workerGroup + '\'' + + ", environmentCode='" + environmentCode + '\'' + '}'; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index dd98fcac9c..693f019e4a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -219,6 +219,11 @@ public class ProcessInstance { */ private String workerGroup; + /** + * environment code + */ + private Long environmentCode; + /** * process timeout for warning */ @@ -490,6 +495,14 @@ public class ProcessInstance { this.executorName = executorName; } + public Long getEnvironmentCode() { + return this.environmentCode; + } + + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + /** * add command to history * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java index e1b4c90ea4..0d2ea9eca8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java @@ -140,6 +140,11 @@ public class Schedule { */ private String workerGroup; + /** + * environment code + */ + private Long environmentCode; + public int getWarningGroupId() { return warningGroupId; } @@ -287,6 +292,14 @@ public class Schedule { this.workerGroup = workerGroup; } + public Long getEnvironmentCode() { + return this.environmentCode; + } + + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + @Override public String toString() { return "Schedule{" @@ -309,6 +322,7 @@ public class Schedule { + ", warningGroupId=" + warningGroupId + ", processInstancePriority=" + processInstancePriority + ", workerGroup='" + workerGroup + '\'' + + ", environmentCode='" + environmentCode + '\'' + '}'; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java index 1e479c82d2..2b27fc4631 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java @@ -134,6 +134,11 @@ public class TaskDefinition { */ private String workerGroup; + /** + * environment code + */ + private Long environmentCode; + /** * fail retry times */ @@ -419,6 +424,14 @@ public class TaskDefinition { this.modifyBy = modifyBy; } + public Long getEnvironmentCode() { + return this.environmentCode; + } + + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + @Override public boolean equals(Object o) { if (o == null) { @@ -461,6 +474,7 @@ public class TaskDefinition { + ", projectName='" + projectName + '\'' + ", workerGroup='" + workerGroup + '\'' + ", failRetryTimes=" + failRetryTimes + + ", environmentCode='" + environmentCode + '\'' + ", failRetryInterval=" + failRetryInterval + ", timeoutFlag=" + timeoutFlag + ", timeoutNotifyStrategy=" + timeoutNotifyStrategy diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinitionLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinitionLog.java index 1fb2060f5f..55b8bfbcd2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinitionLog.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinitionLog.java @@ -53,6 +53,7 @@ public class TaskDefinitionLog extends TaskDefinition { this.setUserId(taskDefinition.getUserId()); this.setUserName(taskDefinition.getUserName()); this.setWorkerGroup(taskDefinition.getWorkerGroup()); + this.setEnvironmentCode(taskDefinition.getEnvironmentCode()); this.setProjectCode(taskDefinition.getProjectCode()); this.setProjectName(taskDefinition.getProjectName()); this.setResourceIds(taskDefinition.getResourceIds()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 2be4ad659e..47c6082f54 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -220,6 +220,15 @@ public class TaskInstance implements Serializable { */ private String workerGroup; + /** + * environment code + */ + private Long environmentCode; + + /** + * environment config + */ + private String environmentConfig; /** * executor id @@ -421,6 +430,22 @@ public class TaskInstance implements Serializable { this.appLink = appLink; } + public Long getEnvironmentCode() { + return this.environmentCode; + } + + public void setEnvironmentCode(Long environmentCode) { + this.environmentCode = environmentCode; + } + + public String getEnvironmentConfig() { + return this.environmentConfig; + } + + public void setEnvironmentConfig(String environmentConfig) { + this.environmentConfig = environmentConfig; + } + public DependentParameters getDependency() { if (this.dependency == null) { Map taskParamsMap = JSONUtils.toMap(this.getTaskParams(), String.class, Object.class); @@ -623,6 +648,8 @@ public class TaskInstance implements Serializable { + ", processInstancePriority=" + processInstancePriority + ", dependentResult='" + dependentResult + '\'' + ", workerGroup='" + workerGroup + '\'' + + ", environmentCode=" + environmentCode + + ", environmentConfig='" + environmentConfig + '\'' + ", executorId=" + executorId + ", executorName='" + executorName + '\'' + ", delayTime=" + delayTime diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java new file mode 100644 index 0000000000..5bde2a3443 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.Environment; + +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + +/** + * environment mapper interface + */ +public interface EnvironmentMapper extends BaseMapper { + + /** + * query environment by name + * + * @param name name + * @return environment + */ + Environment queryByEnvironmentName(@Param("environmentName") String name); + + /** + * query environment by code + * + * @param environmentCode environmentCode + * @return environment + */ + Environment queryByEnvironmentCode(@Param("environmentCode") Long environmentCode); + + /** + * query all environment list + * @return environment list + */ + List queryAllEnvironmentList(); + + /** + * environment page + * @param page page + * @param searchName searchName + * @return environment IPage + */ + IPage queryEnvironmentListPaging(IPage page, @Param("searchName") String searchName); + + /** + * delete environment by code + * + * @param code code + * @return int + */ + int deleteByCode(@Param("code") Long code); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.java new file mode 100644 index 0000000000..44375368f2 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; + +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * environment worker group relation mapper interface + */ +public interface EnvironmentWorkerGroupRelationMapper extends BaseMapper { + + /** + * environment worker group relation by environmentCode + * + * @param environmentCode environmentCode + * @return EnvironmentWorkerGroupRelation list + */ + List queryByEnvironmentCode(@Param("environmentCode") Long environmentCode); + + /** + * environment worker group relation by workerGroupName + * + * @param workerGroupName workerGroupName + * @return EnvironmentWorkerGroupRelation list + */ + List queryByWorkerGroupName(@Param("workerGroupName") String workerGroupName); + + /** + * delete environment worker group relation by processCode + * + * @param environmentCode environmentCode + * @param workerGroupName workerGroupName + * @return int + */ + int deleteByCode(@Param("environmentCode") Long environmentCode, @Param("workerGroupName") String workerGroupName); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java index 1c0f002567..14eceffa72 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.upgrade.shell; import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager; diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index fa02dbc11b..d0e9141bc5 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -21,7 +21,7 @@ + select + + from t_ds_environment + WHERE name = #{environmentName} + + + + + + delete from t_ds_environment where code = #{code} + + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.xml new file mode 100644 index 0000000000..7ea959d601 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapper.xml @@ -0,0 +1,40 @@ + + + + + + + id, environment_code, worker_group, operator, create_time, update_time + + + + + delete from t_ds_environment_worker_group_relation + WHERE environment_code = #{environmentCode} and worker_group = #{workerGroupName} + + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml index db56301990..f1b074db6c 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml @@ -23,7 +23,7 @@ command_type, command_param, task_depend_type, max_try_times, failure_strategy, warning_type, warning_group_id, schedule_time, command_start_time, global_params, flag, update_time, is_sub_process, executor_id, history_cmd, - process_instance_priority, worker_group, timeout, tenant_id, var_pool + process_instance_priority, worker_group,environment_code, timeout, tenant_id, var_pool diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.xml index a92558946f..458b096cdd 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.xml @@ -20,7 +20,7 @@ id, code, `name`, version, description, project_code, user_id, task_type, task_params, flag, task_priority, - worker_group, fail_retry_times, fail_retry_interval, timeout_flag, timeout_notify_strategy, timeout, delay_time, + worker_group, environment_code, fail_retry_times, fail_retry_interval, timeout_flag, timeout_notify_strategy, timeout, delay_time, resource_ids, operator, operate_time, create_time, update_time diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml index b9f834588a..dab824d3b4 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml @@ -21,13 +21,13 @@ id, name, task_type, process_instance_id, task_code, task_definition_version, state, submit_time, start_time, end_time, host, execute_path, log_path, alert_flag, retry_times, pid, app_link, - flag, retry_interval, max_retry_times, task_instance_priority, worker_group, executor_id, + flag, retry_interval, max_retry_times, task_instance_priority, worker_group,environment_code , executor_id, first_submit_time, delay_time, task_params, var_pool ${alias}.id, ${alias}.name, ${alias}.task_type, ${alias}.task_code, ${alias}.task_definition_version, ${alias}.process_instance_id, ${alias}.state, ${alias}.submit_time, ${alias}.start_time, ${alias}.end_time, ${alias}.host, ${alias}.execute_path, ${alias}.log_path, ${alias}.alert_flag, ${alias}.retry_times, ${alias}.pid, ${alias}.app_link, - ${alias}.flag, ${alias}.retry_interval, ${alias}.max_retry_times, ${alias}.task_instance_priority, ${alias}.worker_group, ${alias}.executor_id, + ${alias}.flag, ${alias}.retry_interval, ${alias}.max_retry_times, ${alias}.task_instance_priority, ${alias}.worker_group,${alias}.environment_code , ${alias}.executor_id, ${alias}.first_submit_time, ${alias}.delay_time, ${alias}.task_params, ${alias}.var_pool diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapperTest.java new file mode 100644 index 0000000000..80e5695543 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapperTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.Environment; + +import java.util.Date; +import java.util.List; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +@RunWith(SpringRunner.class) +@SpringBootTest +@Transactional +@Rollback(false) +public class EnvironmentMapperTest { + + @Autowired + EnvironmentMapper environmentMapper; + + /** + * insert + * + * @return Environment + */ + private Environment insertOne() { + //insertOne + Environment environment = new Environment(); + environment.setName("testEnv"); + environment.setCode(1L); + environment.setOperator(1); + environment.setConfig(getConfig()); + environment.setDescription(getDesc()); + environment.setCreateTime(new Date()); + environment.setUpdateTime(new Date()); + environmentMapper.insert(environment); + return environment; + } + + @Before + public void setUp() { + clearTestData(); + } + + @After + public void after() { + clearTestData(); + } + + public void clearTestData() { + environmentMapper.queryAllEnvironmentList().stream().forEach(environment -> { + environmentMapper.deleteByCode(environment.getCode()); + }); + } + + /** + * test update + */ + @Test + public void testUpdate() { + //insertOne + Environment environment = insertOne(); + environment.setDescription("new description info"); + //update + int update = environmentMapper.updateById(environment); + Assert.assertEquals(update, 1); + } + + /** + * test delete + */ + @Test + public void testDelete() { + Environment environment = insertOne(); + int delete = environmentMapper.deleteById(environment.getId()); + Assert.assertEquals(delete, 1); + } + + /** + * test query + */ + @Test + public void testQuery() { + insertOne(); + //query + List environments = environmentMapper.selectList(null); + Assert.assertEquals(environments.size(), 1); + } + + /** + * test query environment by name + */ + @Test + public void testQueryByEnvironmentName() { + Environment entity = insertOne(); + Environment environment = environmentMapper.queryByEnvironmentName(entity.getName()); + Assert.assertEquals(entity.toString(),environment.toString()); + } + + /** + * test query environment by code + */ + @Test + public void testQueryByEnvironmentCode() { + Environment entity = insertOne(); + Environment environment = environmentMapper.queryByEnvironmentCode(entity.getCode()); + Assert.assertEquals(entity.toString(),environment.toString()); + } + + /** + * test query all environments + */ + @Test + public void testQueryAllEnvironmentList() { + Environment entity = insertOne(); + List environments = environmentMapper.queryAllEnvironmentList(); + Assert.assertEquals(environments.size(), 1); + Assert.assertEquals(entity.toString(),environments.get(0).toString()); + } + + /** + * test query environment list paging + */ + @Test + public void testQueryEnvironmentListPaging() { + Environment entity = insertOne(); + Page page = new Page<>(1, 10); + IPage environmentIPage = environmentMapper.queryEnvironmentListPaging(page,""); + List environmentList = environmentIPage.getRecords(); + Assert.assertEquals(environmentList.size(), 1); + + environmentIPage = environmentMapper.queryEnvironmentListPaging(page,"abc"); + environmentList = environmentIPage.getRecords(); + Assert.assertEquals(environmentList.size(), 0); + } + + /** + * test query all environments + */ + @Test + public void testDeleteByCode() { + Environment entity = insertOne(); + int delete = environmentMapper.deleteByCode(entity.getCode()); + Assert.assertEquals(delete, 1); + } + + private String getDesc() { + return "create an environment to test "; + } + + /** + * create an environment config + */ + private String getConfig() { + return "export HADOOP_HOME=/opt/hadoop-2.6.5\n" + + "export HADOOP_CONF_DIR=/etc/hadoop/conf\n" + + "export SPARK_HOME1=/opt/soft/spark1\n" + + "export SPARK_HOME2=/opt/soft/spark2\n" + + "export PYTHON_HOME=/opt/soft/python\n" + + "export JAVA_HOME=/opt/java/jdk1.8.0_181-amd64\n" + + "export HIVE_HOME=/opt/soft/hive\n" + + "export FLINK_HOME=/opt/soft/flink\n" + + "export DATAX_HOME=/opt/soft/datax\n" + + "export YARN_CONF_DIR=\"/etc/hadoop/conf\"\n" + + "\n" + + "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH\n" + + "\n" + + "export HADOOP_CLASSPATH=`hadoop classpath`\n" + + "\n" + + "#echo \"HADOOP_CLASSPATH=\"$HADOOP_CLASSPATH"; + } +} \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java new file mode 100644 index 0000000000..331b070778 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; + +import java.util.Date; +import java.util.List; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; + +@RunWith(SpringRunner.class) +@SpringBootTest +@Transactional +@Rollback(true) +public class EnvironmentWorkerGroupRelationMapperTest { + + @Autowired + EnvironmentWorkerGroupRelationMapper environmentWorkerGroupRelationMapper; + + @Before + public void setUp() { + clearTestData(); + } + + @After + public void after() { + clearTestData(); + } + + public void clearTestData() { + environmentWorkerGroupRelationMapper.selectList(null).stream().forEach(environment -> { + environmentWorkerGroupRelationMapper.deleteById(environment.getId()); + }); + } + + /** + * insert + * + * @return ProcessDefinition + */ + private EnvironmentWorkerGroupRelation insertOne() { + //insertOne + EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); + relation.setEnvironmentCode(1L); + relation.setWorkerGroup("default"); + relation.setOperator(1); + relation.setUpdateTime(new Date()); + relation.setCreateTime(new Date()); + environmentWorkerGroupRelationMapper.insert(relation); + return relation; + } + + /** + * test query + */ + @Test + public void testQuery() { + insertOne(); + //query + List relations = environmentWorkerGroupRelationMapper.selectList(null); + Assert.assertEquals(relations.size(), 1); + } + + @Test + public void testQueryByEnvironmentCode() { + EnvironmentWorkerGroupRelation relation = insertOne(); + List environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(1L); + Assert.assertNotEquals(environmentWorkerGroupRelations.size(), 0); + } + + @Test + public void testQueryByWorkerGroupName() { + EnvironmentWorkerGroupRelation relation = insertOne(); + List environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByWorkerGroupName("default"); + Assert.assertNotEquals(environmentWorkerGroupRelations.size(), 0); + } + + @Test + public void testDeleteByCode() { + EnvironmentWorkerGroupRelation relation = insertOne(); + int i = environmentWorkerGroupRelationMapper.deleteByCode(1L, "default"); + Assert.assertNotEquals(i, 0); + } +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapperTest.java index cfa88c5487..cc3af636e0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapperTest.java @@ -65,6 +65,8 @@ public class TaskDefinitionLogMapperTest { taskDefinition.setProjectCode(1L); taskDefinition.setTaskType(TaskType.SHELL.getDesc()); taskDefinition.setUserId(userId); + taskDefinition.setEnvironmentCode(1L); + taskDefinition.setWorkerGroup("default"); taskDefinition.setVersion(1); taskDefinition.setCreateTime(new Date()); taskDefinition.setUpdateTime(new Date()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java index 2035a3702b..f30e55a220 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java @@ -63,6 +63,8 @@ public class TaskDefinitionMapperTest { taskDefinition.setTaskType(TaskType.SHELL.getDesc()); taskDefinition.setUserId(userId); taskDefinition.setResourceIds("1"); + taskDefinition.setWorkerGroup("default"); + taskDefinition.setEnvironmentCode(1L); taskDefinition.setVersion(1); taskDefinition.setCreateTime(new Date()); taskDefinition.setUpdateTime(new Date()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java index c1cca3a1bd..f37af08e7d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java @@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; */ public class TaskExecutionContextBuilder { - public static TaskExecutionContextBuilder get(){ + public static TaskExecutionContextBuilder get() { return new TaskExecutionContextBuilder(); } @@ -56,6 +56,7 @@ public class TaskExecutionContextBuilder { taskExecutionContext.setTaskType(taskInstance.getTaskType()); taskExecutionContext.setLogPath(taskInstance.getLogPath()); taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup()); + taskExecutionContext.setEnvironmentConfig(taskInstance.getEnvironmentConfig()); taskExecutionContext.setHost(taskInstance.getHost()); taskExecutionContext.setResources(taskInstance.getResources()); taskExecutionContext.setDelayTime(taskInstance.getDelayTime()); @@ -76,14 +77,13 @@ public class TaskExecutionContextBuilder { return this; } - /** * build processInstance related info * * @param processInstance processInstance * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildProcessInstanceRelatedInfo(ProcessInstance processInstance){ + public TaskExecutionContextBuilder buildProcessInstanceRelatedInfo(ProcessInstance processInstance) { taskExecutionContext.setProcessInstanceId(processInstance.getId()); taskExecutionContext.setScheduleTime(processInstance.getScheduleTime()); taskExecutionContext.setGlobalParams(processInstance.getGlobalParams()); @@ -100,33 +100,31 @@ public class TaskExecutionContextBuilder { * @param processDefinition processDefinition * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildProcessDefinitionRelatedInfo(ProcessDefinition processDefinition){ + public TaskExecutionContextBuilder buildProcessDefinitionRelatedInfo(ProcessDefinition processDefinition) { taskExecutionContext.setProcessDefineCode(processDefinition.getCode()); taskExecutionContext.setProcessDefineVersion(processDefinition.getVersion()); taskExecutionContext.setProjectCode(processDefinition.getProjectCode()); return this; } - /** * build SQLTask related info * * @param sqlTaskExecutionContext sqlTaskExecutionContext * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildSQLTaskRelatedInfo(SQLTaskExecutionContext sqlTaskExecutionContext){ + public TaskExecutionContextBuilder buildSQLTaskRelatedInfo(SQLTaskExecutionContext sqlTaskExecutionContext) { taskExecutionContext.setSqlTaskExecutionContext(sqlTaskExecutionContext); return this; } - /** * build DataxTask related info * * @param dataxTaskExecutionContext dataxTaskExecutionContext * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildDataxTaskRelatedInfo(DataxTaskExecutionContext dataxTaskExecutionContext){ + public TaskExecutionContextBuilder buildDataxTaskRelatedInfo(DataxTaskExecutionContext dataxTaskExecutionContext) { taskExecutionContext.setDataxTaskExecutionContext(dataxTaskExecutionContext); return this; } @@ -137,7 +135,7 @@ public class TaskExecutionContextBuilder { * @param procedureTaskExecutionContext procedureTaskExecutionContext * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildProcedureTaskRelatedInfo(ProcedureTaskExecutionContext procedureTaskExecutionContext){ + public TaskExecutionContextBuilder buildProcedureTaskRelatedInfo(ProcedureTaskExecutionContext procedureTaskExecutionContext) { taskExecutionContext.setProcedureTaskExecutionContext(procedureTaskExecutionContext); return this; } @@ -148,18 +146,17 @@ public class TaskExecutionContextBuilder { * @param sqoopTaskExecutionContext sqoopTaskExecutionContext * @return TaskExecutionContextBuilder */ - public TaskExecutionContextBuilder buildSqoopTaskRelatedInfo(SqoopTaskExecutionContext sqoopTaskExecutionContext){ + public TaskExecutionContextBuilder buildSqoopTaskRelatedInfo(SqoopTaskExecutionContext sqoopTaskExecutionContext) { taskExecutionContext.setSqoopTaskExecutionContext(sqoopTaskExecutionContext); return this; } - /** * create * * @return taskExecutionContext */ - public TaskExecutionContext create(){ + public TaskExecutionContext create() { return taskExecutionContext; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java index f50b6383b8..c32c38347f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java @@ -157,6 +157,12 @@ public class TaskExecutionContext implements Serializable { */ private String envFile; + /** + * environmentConfig + */ + private String environmentConfig; + + /** * definedParams */ @@ -424,6 +430,14 @@ public class TaskExecutionContext implements Serializable { this.envFile = envFile; } + public String getEnvironmentConfig() { + return environmentConfig; + } + + public void setEnvironmentConfig(String config) { + this.environmentConfig = config; + } + public Map getDefinedParams() { return definedParams; } @@ -566,6 +580,7 @@ public class TaskExecutionContext implements Serializable { + ", taskTimeoutStrategy=" + taskTimeoutStrategy + ", taskTimeout=" + taskTimeout + ", workerGroup='" + workerGroup + '\'' + + ", environmentConfig='" + environmentConfig + '\'' + ", delayTime=" + delayTime + ", resources=" + resources + ", sqlTaskExecutionContext=" + sqlTaskExecutionContext diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java index 8ae3481f74..ed369246ca 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java @@ -50,6 +50,7 @@ import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProjectUser; @@ -77,6 +78,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; @@ -723,10 +725,23 @@ public class WorkflowExecuteThread implements Runnable { String processWorkerGroup = processInstance.getWorkerGroup(); processWorkerGroup = StringUtils.isBlank(processWorkerGroup) ? DEFAULT_WORKER_GROUP : processWorkerGroup; String taskWorkerGroup = StringUtils.isBlank(taskNode.getWorkerGroup()) ? processWorkerGroup : taskNode.getWorkerGroup(); + + Long processEnvironmentCode = Objects.isNull(processInstance.getEnvironmentCode()) ? -1 : processInstance.getEnvironmentCode(); + Long taskEnvironmentCode = Objects.isNull(taskNode.getEnvironmentCode()) ? processEnvironmentCode : taskNode.getEnvironmentCode(); + if (!processWorkerGroup.equals(DEFAULT_WORKER_GROUP) && taskWorkerGroup.equals(DEFAULT_WORKER_GROUP)) { taskInstance.setWorkerGroup(processWorkerGroup); + taskInstance.setEnvironmentCode(processEnvironmentCode); } else { taskInstance.setWorkerGroup(taskWorkerGroup); + taskInstance.setEnvironmentCode(taskEnvironmentCode); + } + + if (!taskInstance.getEnvironmentCode().equals(-1L)) { + Environment environment = processService.findEnvironmentByCode(taskInstance.getEnvironmentCode()); + if (Objects.nonNull(environment) && StringUtils.isNotEmpty(environment.getConfig())) { + taskInstance.setEnvironmentConfig(environment.getConfig()); + } } // delay execution time taskInstance.setDelayTime(taskNode.getDelayTime()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java index 2aa80f1ace..d8c1b4af96 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.task; +import java.util.Arrays; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; @@ -122,6 +123,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { @Override protected String commandInterpreter() { String pythonHome = getPythonHome(taskExecutionContext.getEnvFile()); + + if (StringUtils.isNotBlank(taskExecutionContext.getEnvironmentConfig())) { + pythonHome = getPythonHomeFromEnvironmentConfig(taskExecutionContext.getEnvironmentConfig()); + } + logger.info("PYTHON_HOME={}",pythonHome); return getPythonCommand(pythonHome); } @@ -185,4 +191,24 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { return null; } + /** + * get python home from the environment config + * + * @param environmentConfig env config + * @return python home + */ + public static String getPythonHomeFromEnvironmentConfig(String environmentConfig) { + String[] lines = environmentConfig.split("\n"); + + String pythonHomeConfig = Arrays.stream(lines).filter(line -> line.contains(Constants.PYTHON_HOME)).findFirst().get(); + + if (StringUtils.isEmpty(pythonHomeConfig)) { + return null; + } + String[] arrs = pythonHomeConfig.split(Constants.EQUAL_SIGN); + if (arrs.length == 2) { + return arrs[1]; + } + return null; + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java index bfc1ae4240..12007a68bc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java @@ -21,13 +21,13 @@ import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.util.Strings; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Consumer; @@ -97,18 +97,25 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { if (OSUtils.isWindows()) { sb.append("@echo off\n"); sb.append("cd /d %~dp0\n"); - if (taskExecutionContext.getEnvFile() != null) { - sb.append("call ").append(taskExecutionContext.getEnvFile()).append("\n"); + if (Strings.isNotBlank(taskExecutionContext.getEnvironmentConfig())) { + sb.append(taskExecutionContext.getEnvironmentConfig()).append("\n"); + } else { + if (taskExecutionContext.getEnvFile() != null) { + sb.append("call ").append(taskExecutionContext.getEnvFile()).append("\n"); + } } } else { sb.append("#!/bin/sh\n"); sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); sb.append("cd $BASEDIR\n"); - if (taskExecutionContext.getEnvFile() != null) { - sb.append("source ").append(taskExecutionContext.getEnvFile()).append("\n"); + if (Strings.isNotBlank(taskExecutionContext.getEnvironmentConfig())) { + sb.append(taskExecutionContext.getEnvironmentConfig()).append("\n"); + } else { + if (taskExecutionContext.getEnvFile() != null) { + sb.append("source ").append(taskExecutionContext.getEnvFile()).append("\n"); + } } } - sb.append(execCommand); logger.info("command : {}", sb.toString()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java index f4e9080a68..c2a79449d2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java @@ -37,6 +37,28 @@ public class PythonCommandExecutorTest { Assert.assertNotNull(pythonHome); } + @Test + public void testGetPythonHomeFromEnvironmentConfig() { + String environmentConfig = "export HADOOP_HOME=/opt/hadoop-2.6.5\n" + + "export HADOOP_CONF_DIR=/etc/hadoop/conf\n" + + "export SPARK_HOME=/opt/soft/spark\n" + + "\n" + + "export PYTHON_HOME=/opt/soft/python\n" + + "export JAVA_HOME=/opt/java/jdk1.8.0_181-amd64\n" + + "export HIVE_HOME=/opt/soft/hive\n" + + "export FLINK_HOME=/opt/soft/flink\n" + + "export DATAX_HOME=/opt/soft/datax\n" + + "export YARN_CONF_DIR=/etc/hadoop/conf\n" + + "export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH\n" + + "export HADOOP_CLASSPATH=`hadoop classpath`"; + + String expected = "/opt/soft/python"; + + String pythonHome = PythonCommandExecutor.getPythonHomeFromEnvironmentConfig(environmentConfig); + logger.info(pythonHome); + Assert.assertEquals(expected,pythonHome); + } + @Test public void testGetPythonCommand() { String pythonCommand = PythonCommandExecutor.getPythonCommand(null); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 33a10fa7be..702912f60b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -64,6 +64,7 @@ import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.ErrorCommand; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; @@ -83,6 +84,7 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; @@ -125,6 +127,8 @@ import org.springframework.transaction.annotation.Transactional; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.facebook.presto.jdbc.internal.guava.collect.Lists; +import com.cronutils.model.Cron; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; /** @@ -198,6 +202,9 @@ public class ProcessService { @Autowired private ProcessTaskRelationLogMapper processTaskRelationLogMapper; + @Autowired + private EnvironmentMapper environmentMapper; + /** * handle Command (construct ProcessInstance from Command) , wrapped in transaction * @@ -296,6 +303,10 @@ public class ProcessService { /** * get command page + * + * @param pageSize + * @param pageNumber + * @return */ public List findCommandPage(int pageSize, int pageNumber) { Page commandPage = new Page<>(pageNumber, pageSize); @@ -526,6 +537,8 @@ public class ProcessService { } return; } + ProcessDefinition processDefinition = this.findProcessDefinition(processInstance.getProcessDefinitionCode(), + processInstance.getProcessDefinitionVersion()); Map cmdParam = new HashMap<>(); cmdParam.put(Constants.CMD_PARAM_RECOVERY_WAITING_THREAD, String.valueOf(processInstance.getId())); // process instance quit by "waiting thread" state @@ -541,6 +554,7 @@ public class ProcessService { processInstance.getWarningGroupId(), processInstance.getScheduleTime(), processInstance.getWorkerGroup(), + processInstance.getEnvironmentCode(), processInstance.getProcessInstancePriority() ); saveCommand(command); @@ -631,6 +645,7 @@ public class ProcessService { processInstance.setProcessInstancePriority(command.getProcessInstancePriority()); String workerGroup = StringUtils.isBlank(command.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP : command.getWorkerGroup(); processInstance.setWorkerGroup(workerGroup); + processInstance.setEnvironmentCode(Objects.isNull(command.getEnvironmentCode()) ? -1 : command.getEnvironmentCode()); processInstance.setTimeout(processDefinition.getTimeout()); processInstance.setTenantId(processDefinition.getTenantId()); return processInstance; @@ -688,6 +703,21 @@ public class ProcessService { return tenant; } + /** + * get an environment + * use the code of the environment to find a environment. + * + * @param environmentCode environmentCode + * @return Environment + */ + public Environment findEnvironmentByCode(Long environmentCode) { + Environment environment = null; + if (environmentCode >= 0) { + environment = environmentMapper.queryByEnvironmentCode(environmentCode); + } + return environment; + } + /** * check command parameters is valid * @@ -1034,6 +1064,11 @@ public class ProcessService { /** * retry submit task to db + * + * @param taskInstance + * @param commitRetryTimes + * @param commitInterval + * @return */ public TaskInstance submitTask(TaskInstance taskInstance, int commitRetryTimes, int commitInterval) { @@ -1084,8 +1119,8 @@ public class ProcessService { createSubWorkProcess(processInstance, task); } - logger.info("end submit task to db successfully:{} state:{} complete, instance id:{} state: {} ", - taskInstance.getName(), task.getState(), processInstance.getId(), processInstance.getState()); + logger.info("end submit task to db successfully:{} {} state:{} complete, instance id:{} state: {} ", + taskInstance.getId(), taskInstance.getName(), task.getState(), processInstance.getId(), processInstance.getState()); return task; } @@ -1242,6 +1277,7 @@ public class ProcessService { parentProcessInstance.getWarningGroupId(), parentProcessInstance.getScheduleTime(), task.getWorkerGroup(), + task.getEnvironmentCode(), parentProcessInstance.getProcessInstancePriority() ); } @@ -2111,6 +2147,44 @@ public class ProcessService { return result; } + /** + * update task definition + */ + public int updateTaskDefinition(User operator, Long projectCode, TaskNode taskNode, TaskDefinition taskDefinition) { + Integer version = taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode()); + Date now = new Date(); + taskDefinition.setProjectCode(projectCode); + taskDefinition.setUserId(operator.getId()); + taskDefinition.setVersion(version == null || version == 0 ? 1 : version + 1); + taskDefinition.setUpdateTime(now); + setTaskFromTaskNode(taskNode, taskDefinition); + int update = taskDefinitionMapper.updateById(taskDefinition); + // save task definition log + TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog(taskDefinition); + taskDefinitionLog.setOperator(operator.getId()); + taskDefinitionLog.setOperateTime(now); + int insert = taskDefinitionLogMapper.insert(taskDefinitionLog); + return insert & update; + } + + private void setTaskFromTaskNode(TaskNode taskNode, TaskDefinition taskDefinition) { + taskDefinition.setName(taskNode.getName()); + taskDefinition.setDescription(taskNode.getDesc()); + taskDefinition.setTaskType(taskNode.getType().toUpperCase()); + taskDefinition.setTaskParams(taskNode.getTaskParams()); + taskDefinition.setFlag(taskNode.isForbidden() ? Flag.NO : Flag.YES); + taskDefinition.setTaskPriority(taskNode.getTaskInstancePriority()); + taskDefinition.setWorkerGroup(taskNode.getWorkerGroup()); + taskDefinition.setEnvironmentCode(Objects.isNull(taskNode.getEnvironmentCode()) ? -1 : taskNode.getEnvironmentCode()); + taskDefinition.setFailRetryTimes(taskNode.getMaxRetryTimes()); + taskDefinition.setFailRetryInterval(taskNode.getRetryInterval()); + taskDefinition.setTimeoutFlag(taskNode.getTaskTimeoutParameter().getEnable() ? TimeoutFlag.OPEN : TimeoutFlag.CLOSE); + taskDefinition.setTimeoutNotifyStrategy(taskNode.getTaskTimeoutParameter().getStrategy()); + taskDefinition.setTimeout(taskNode.getTaskTimeoutParameter().getInterval()); + taskDefinition.setDelayTime(taskNode.getDelayTime()); + taskDefinition.setResourceIds(getResourceIds(taskDefinition)); + } + /** * get resource ids * @@ -2357,6 +2431,7 @@ public class ProcessService { v.setParams(JSONUtils.toJsonString(taskParamsMap)); v.setTaskInstancePriority(taskDefinitionLog.getTaskPriority()); v.setWorkerGroup(taskDefinitionLog.getWorkerGroup()); + v.setEnvironmentCode(taskDefinitionLog.getEnvironmentCode()); v.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN, taskDefinitionLog.getTimeoutNotifyStrategy(), taskDefinitionLog.getTimeout()))); diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index f24d43a6c7..12dd8d99b5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + .dag-chart { width: 100%; height: calc(100vh - 100px); diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/relatedEnvironment.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/relatedEnvironment.vue new file mode 100644 index 0000000000..a75fe5c6a3 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/relatedEnvironment.vue @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 68e97fe496..cb4a6f3525 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -77,8 +77,18 @@ - {{$t('Worker group')}} - + + + + + +
{{$t('Worker group')}}
+
+ + + + {{$t('Environment Name')}} +
@@ -95,7 +105,7 @@ - +
{{$t('Delay execution time')}}
@@ -258,6 +268,13 @@ :backfill-item="backfillItem" :pre-node="nodeData.preNode"> +